{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Tensorflow for autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# load library\n",
    "import os\n",
    "import sys\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(42)\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.rcParams[\"axes.labelsize\"] = 14\n",
    "plt.rcParams[\"xtick.labelsize\"] = 12\n",
    "plt.rcParams[\"ytick.labelsize\"] = 12\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "ID = \"autoencoders\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# utility functions\n",
    "# save figures\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", ID, fig_id + \".png\")\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format=\"png\", dpi=300)\n",
    "\n",
    "def plot_image(image, shape=[28, 28]):\n",
    "    plt.imshow(image.reshape(shape), cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")\n",
    "    \n",
    "def plot_multi_images(images, n_rows, n_cols, pad=2):\n",
    "    images -= images.min()\n",
    "    w, h = images.shape[1:]\n",
    "    image = np.zeros(((w + pad) * n_rows + pad, (h + pad) * n_cols + pad))\n",
    "    for y in range(n_rows):\n",
    "        for x in range(n_cols):\n",
    "            image[(y*(w+pad)+pad):(y*(w+pad)+pad+h), (x*(h+pad)+pad):(x*(h+pad)+pad+h)] = images[y*n_cols+x]\n",
    "    plt.imshow(image, cmap=\"Greys\", interpolation=\"nearest\")\n",
    "    plt.axis(\"off\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## PCA with a linear Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# 3D dataset\n",
    "np.random.seed(2017)\n",
    "m = 100\n",
    "w1, w2 = 0.1, 0.3\n",
    "noise = 0.1\n",
    "\n",
    "angles = np.random.rand(m) * 3 * np.pi / 2 - 0.5\n",
    "X_train = np.empty((m, 3))\n",
    "X_train[:, 0] = np.cos(angles) + np.sin(angles)/2 + noise * np.random.randn(m) / 2\n",
    "X_train[:, 1] = np.sin(angles) * 0.7 + noise * np.random.randn(m) / 2\n",
    "X_train[:, 2] = X_train[:, 0] * w1 + X_train[:, 1] * w2 + noise * np.random.randn(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# normalize the data\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# PCA\n",
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 3\n",
    "n_hidden = 2 # encoder\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "h = tf.layers.dense(X, n_hidden, activation=None)  # linear transform\n",
    "outputs = tf.layers.dense(h, n_outputs, activation=None)\n",
    "\n",
    "mse_loss = tf.reduce_mean(tf.square(X - outputs))\n",
    "\n",
    "train_op = tf.train.AdamOptimizer(learning_rate).minimize(mse_loss)\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 mse: 1.77635\n",
      "Epoch 1000 mse: 0.0443353\n",
      "Epoch 2000 mse: 0.043948\n",
      "Epoch 3000 mse: 0.0439479\n",
      "Epoch 4000 mse: 0.0439479\n",
      "Epoch 5000 mse: 0.0439501\n",
      "Epoch 6000 mse: 0.043948\n",
      "Epoch 7000 mse: 0.043948\n",
      "Epoch 8000 mse: 0.043948\n",
      "Epoch 9000 mse: 0.0439479\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(n_epochs):\n",
    "        _, mse = sess.run([train_op, mse_loss], feed_dict={X: X_train})\n",
    "        if epoch % 1000 == 0:\n",
    "            print(\"Epoch\", epoch, \"mse:\", mse)\n",
    "    encoded_output = sess.run(h, feed_dict={X: X_train})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure linear_autoencoder_pca_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARgAAADQCAYAAADcQn7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEsVJREFUeJzt3W+MHdV5x/Hvs39wEAlSWJB5ZfyiIKWtGwp+syUVC1ip\ngqIkwlKpGmMFaE34UwmaVoqlWEJQ2WpeRBYBIiwZsAlQ0UAQCVRVFWUVqDeiixJeIEVELZhULY1x\nmwYTsPHu0xdzb319ff/M3J0z58zM7yOtrvfe2ZnjO/c+5zl/5oy5OyIiIUzFLoCINJcCjIgEowAj\nIsEowIhIMAowIhKMAoyIBKMAIyLBKMCISDAKMCISzEzsAkzi/PPP940bN8YuhkhrvfLKK++4+wXj\ntqtlgNm4cSPLy8uxiyHSWmZ2OM92aiKJSDAKMCISjAKMiASjACPSEktLsGdP9liVWnbyikgxS0tw\nzTVw4gScdRb84AcwPx/+uMpgRFpgcTELLisr2ePiYjXHVYARaYGFhSxzmZ7OHhcWqjmumkgiLTA/\nnzWLFhez4FJF8wgUYERaY36+usDSpSaSiASjACMiwSjAiEgwyQQYM7vYzD4ws2/HLouIlCOZAAM8\nAPxL7EKIpC7GjNxJJTGKZGZ/AvwKOAT8VuTiiCQr1ozcSUXPYMzsXOAe4C/HbLfDzJbNbPnIkSPV\nFE6kInmzklgzcieVQgZzL7Df3f/dzIZu5O77gH0Amzdv1g21pTGKZCXdGbndbauakTupqAHGzC4F\ntgC/H7McIjENykqGBZhYM3InFTuDWQA2Am91spePAtNm9tvuflnEcolUpmhWEmNG7qRiB5h9wN/1\n/P5XZAHn1iilEYlgkqxkaenU9nD63/a+FjsQRQ0w7v4b4Dfd383sGPCBu6sXV1qlSFbS22czMwPu\ncPIkmMEVV8CPf5w1t1IYZYqdwZzG3e+OXQaR1PX22ayuZs95Z9jjRz86td2JE3DwYNxsJqkAIyLj\n9fbZzMxk2cvKyunbmGWvPfxw3Gwm+jwYESmm22dz773wwx/Cgw9mC0l1zc7CLbfAjTdmwSXmnBll\nMCKJG9Rp29tnMz8PmzZlzSGA7dtPdfYeOBB3zowCjEjC8k7CG9RJnMKcGQUYkYT1T8Ir2mkbe86M\nAoxIRQY1dcbNWent0J2ehv37s07d2dnRM35ToQAjUoFBTR0Y3/zpbea8/DI8+2z2fDebST3AaBRJ\npAKDrjfKe2X0/Dzs3AkXXnj682+/HbbMZVCAEanAoPsSFb1X0fbtWdOo64UX0l90Sk0kkQoMG9Ep\nMsozPw833wwPPZTN3F1ZSb8fxtzrt7TK5s2bfXl5OXYxRCqXyop2ZvaKu28et50yGJEaSWFuSxEK\nMCIBhFwyIfbcliIUYERKlkozZpgq14tRgBEpWZElMKtWdfDTMLVIyYoOP1ep6rsSKIMRKVnKHbFV\n35VAAUYkgFQ7YqsOfgowIi1TZfBTH4yIBKMAI9IyeW9TWwY1kURaRMPUIhJM1cPUCjAiLVL1HJ2o\nAcbM1pnZfjM7bGbvmtlPzewzMcsk0mS9tzyp4hKG2H0wM8AvgCuBt4BrgafMbJO7vxmzYCJNVeUw\ndex7U78H3N3z1PfN7A3gcuDNGGUSkfIk1QdjZuuBS4DXBry2w8yWzWz5yJEj1RcuAVUOL4qUIXYT\n6f+Z2SzwOHDA3X/W/7q77wP2QbaiXcXFiy71JQBEBkkigzGzKeAx4ARwR+TiJKnq4UVptqqy4egZ\njJkZsB9YD1zr7h9GLtJEQi/iU/VVsNJcVWbD0QMM8C3gE8AWd38/dmEmUcUJS3kJAKmXYdlwiM9W\n1ABjZhcBtwDHgbezZAaAW9z98WgFK6iqFcxSXQJA6qU/G56bC1dBxh6mPgzY2A0TMqgppOaL1El/\nNhyygkyhiVQbw5pCar5I3fRnw6EqSAWYAkZF+jzNlypXcxfJK2QFmSvAmNlZwDFgdsgm33X360or\nVaLW0hTSPBYZp4wKaNJ9hOrfy5vBzAI3DXj+LuAy4HullShh8/Owdy88/TRs3VrshKR8KwuJr4wK\nKMVKLFeA6Vwz9O3e58zs62TB5Svu/kiAsiVnaQnuvDM7gS++CJs25T+B6giWUcqogCbdRzfrmZuD\no0fLbSYV7oPpTIy7D7gduN3dHyynKOUK0d+xuAjHj8PqavZY5EOgjmAZpYwKaJJ9dLOe7ud6agrW\nrSsv+ykUYDpT+h8iay7d3M1czGwdcD9wDXAB8J/AN939m2svYnGhUsW5uewkQPY4N1fs7zWPRYYp\nowKaZB/drKf3c11mEz53gDGzaeAAcD2wzd2f7NvP28CngX8Dfg/4RzP7L3d/au3FLCZUf8fRo1mE\n70b6o0eHb6sRIymqjAqo6D66WU9vBlNmEz7vKNIs8ATwOeB6d3+m9/VOH82unqd+amYvAJ8CKg8w\nofo75uayEwBZGjlsvyl2tklzraUy6816ovTBdJo/3wG2ANe5+/M5/maWLLj87ZpLOIEQ/R3dDt7V\n1Ww90717h+9XI0ZSlTIqs5BN9zwZzEHgs8CjwMfNbFvf68+5+6/7nrsf+FXnb6Mo+03rbau6w09+\nMnxbjRhJVVKvzEYGmM6IUXcR7i91fnqtAh/r+5tvAPPA1e5+opRSJmBhIctcVlayAPPII7B9++CT\nqREjqUrqldnIAOPuDpybd2dmtpdsJOlqd39njWVLyvw83HQTPPRQFmBOnhxdW2jESKqQemVW2rVI\nZnYfcDVwlbs3ctHc7dvhwIF0awtpp5Qrs1ICTGddl78gW9fljZ51XV5098bc5yj12kIkNaUEmDqu\n6zKplGsLqbcy506lMg9LyzXkkMrJkuYqc+5USvOwkrirQMq6J2vXruxR9ySSEMq8a0RKd6BQgBkj\npZMlzVXmTemrvsH9KGoijZH6PAOpn0FN7jIHEFIajLBsqku9bN682ZeXlys7nvpgpCwp9Y+shZm9\n4u6bx22nDGaA/oAybORIgUeKSn1qf9kUYPrkrWGKbKcgJF1ta3K3KsDkWRowbw2TZ7umpMNSnjL6\nR+pUaUUPMGZ2Htm9qT8NvAPsdPcnyj5O3qUB89YwebZrWzos+eSdrDkokAyqtCDdgBM9wAAPACeA\n9cClwPNm9qq7v1bmQfIuDZi3hsmzXdvSYSnPsOy3v9I6ePD06+NSy5Jj35v6HGAr8Lvufgx4ycye\nA24AvlrmsYosDZi3hhm3XUrDhVIvw7Lf/koL0s6SY2cwlwAn3f31nudeBa7s39DMdgA7ADZs2FD4\nQKGXBhx13JROuNTDsOy3v9KCtK/wjzoPxsz+EPh7d7+w57k/B77o7gvD/q7qeTAiMeTtzI3R6VuX\neTDHOHNBq3OBd8vYeZ1620X6ldVUjyl2gHkdmDGzi939553nPgmsuYO3qiFiBTEJpQmfragBxt3f\nM7NngHvM7M/IRpE+D/zBWvedd57KWucjaJ6LhNCUz1YKV1PfBpwN/BJ4Eri1jCHqcVeUlrEMg660\nllCa8tmK3UTC3f8b+ELZ+x03RFzGJLjenv6ZGXjrrSxQ1bGmkbQ0ZQ5Va6+mLisFXVrKJjs9/HAW\nrOqczko1Uh4dyqsuo0jRlDUJrju7cmUl3clOkoailVHKo0N5tTbAQHknsCnprITTzZg/+CC7rxa0\nozJqdYApiy4JkHG6fX7d4GLWjspIAaYkTUhnJZyFhWwgYGUl+31qCvbubf5nJoVhapFGWlqCPXtO\njSzeeGOWuXQdPRqvbFVpbQaTcg+91N+gUco23nq4lQGmKbMkJY48ldOgeVY7d2bNoqefhq1b2/GZ\na2WAKWOSnTKgdspbOQ0aWVxagjvvzJ578UXYtKn5n51WBphuh9vqavZYNFVdSwbUG5hAQapu8lZO\ng0YW9+xJe3GoEFoZYODUcOEkE5knzYB6A9PMTHZszf6tlyJznvpHFts4X6qVo0jdmbfdL3jRC8km\nvTVnf2D68MP6X8zWNt3M5N57i1cK/X8Lp0aZmqqVGcxaa5JB6W+ePpn+iyN7M5jeMqh/J21rmfPU\n/du2DDS0MsCUMfO290OW98MyaD3VPLelaOIHL0VVBva23NKmlQEGyp15W+TD0n/cEMtISHFVB/a2\n9Me0sg+mbP19MnNzk7etJ+3fkdF6Z9UOUvUCT2vpy6mT1mYwZeq/JUp3rsMkNWHe5pv6afIblp30\nvocxMoo2XL+mANNR5As7aNvuh6WMuQ7jPnhN66cJHSyHZSf976GuiC+fAgzFvrDjtq2iJmxSP00V\nwXLQORk2lb+u72OqFGAo9oUdt20Va8PUsYNwWJZSRbAcdk7q9h7WkQIMxb6webYN3bau2wJXo7KU\nqoLloNG7Or2HdaUAQ7EP27Btq+50rVMH4agsJeYXvU7vYV219q4CZRpVQ2u0J1w/i97beHRXgQoN\nq6HrNtoT6gsbIkup23vbVtECjJmtAx4EtgDnAf8K7HT3fyjzOFXUcsP6Eeo02hP6C1t2c6RO722b\nxcxgZoBfAFcCbwHXAk+Z2SZ3f7OMA1RVyw2roVMc7Yk5mjNp2QZJ8b2VM0ULMO7+HnB3z1PfN7M3\ngMuBN8s4RpVfmkE1dGojFSmM5hQt27Cgk9p7K4Ml0wdjZuuBS4CBN743sx3ADoANGzbk2mfsL01q\nUh3NGVY2GJ2BahQofUkEGDObBR4HDrj7zwZt4+77gH2QjSLl2W/sL01qHZHjAm7ML2ze2bYKKPUS\nLMCY2SJZ/8og/+zun+psNwU8BpwA7ii7HDG/NKl9QWIH3FE027aZggUYd18Yt42ZGbAfWA9c6+4f\nhioPVDtvYmkJXn45+/fUVNwvSP//O6XA0kuzbZsndhPpW8AngC3u/n7IA1XZXFlagquuguPHs9+n\np+PdJrTukwBTDogyXrQFp8zsIuAW4FLgbTM71vn5YojjVbmgUPdYXaur4W8TOmxBpWH/727g2bUr\ne+z/u3ELNInkEXOY+jBgYzcsSZUjSt1jdTOY2dnRxxuVSeTJMiYZfh7VP7RvH9xxR/baunVnZnt1\nyHwkEe5eu5/LL7/cJ3HokPvu3dljaIcOuX/5y9nPqOMdOuR+9tnu09PZY++2o17rtXt3tg1kj7t3\nn3mM/v/3sH0fOuQ+M5PtC9ynpk7fX94ySbMBy57juxq7D6ZSVbbn8x5rVCaRdxRqkuHnYR2oi4tZ\nk65revr0/aU2MiZpa1WASdGo4JC3WTfpaMugwLOwkDWLjh/PRr/uv39tK/aNa06pudVsWq4hAWvt\ng6myPEXKNG7kLrWJiJKflmsIaNAXbC2BYFRzKsYw7bhjltH8y/O61J8CTEGDal1QTTzIuOaUrhVr\nPgWYgobNK1FNfKZxfUOaqdt8CjAFDat1VRMPVlZzS+pJAaagYbWuamKRM2kUSUQKyzuKFO1aJBFp\nPgUYEQlGAUZEglGAEZFgGh9gtK6JSDyNHqbWtS4icTU6g6lyFTsROVOjA0x31u30tGbYisTQ6CaS\nrnURiavRAQZ0rYtITI1uIolIXAowIhKMAoyIBFPLq6nN7AhwOHY51uB84J3YhaiY/s/NcpG7XzBu\no1oGmLozs+U8l7o3if7P7aQmkogEowAjIsEowMSxL3YBItD/uYXUByMiwSiDEZFgFGBEJBgFGBEJ\nRgEmIjO7w8yWzey4mT0auzwhmNl5ZvZdM3vPzA6b2Z/GLlNIbTinRTT+aurE/QfwN8AfAWdHLkso\nDwAngPXApcDzZvaqu78Wt1jBtOGc5qYMJiJ3f8bdnwWOxi5LCGZ2DrAV2OXux9z9JeA54Ia4JQun\n6ee0KAUYCekS4KS7v97z3KvA70Qqj1RMAUZC+ijw677n/hf4WISySAQKMIGY2aKZ+ZCfl2KXryLH\ngHP7njsXeDdCWSQCdfIG4u4LscuQgNeBGTO72N1/3nnuk0BTO3iljzKYiMxsxsw+AkwD02b2ETNr\nTNB39/eAZ4B7zOwcM7sC+DzwWNyShdP0c1qUAkxcXwPeB74KbOv8+2tRS1S+28iGa38JPAnc2uAh\namjHOc1NFzuKSDDKYEQkGAUYEQlGAUZEglGAEZFgFGBEJBgFGBEJRgFGRIJRgBGRYBRgRCQYBRgJ\nwszOMrMTI64ofyZ2GSW81l6EJcHNAjcNeP4u4DLge9UWR2LQtUhSGTP7OvDXwFfc/RuxyyPhKYOR\n4MzMgPuA24Hb3f3ByEWSiqgPRoIysymyezTfBtzcG1zM7I/N7CUzO2Zmb8Yqo4SjDEaCMbNp4ABw\nPbDN3Z/s2+R/gPvJbmlyV8XFkwoowEgQZjYLPAF8Drje3c8YNXL3f+ps+4WKiycVUYCR0pnZOuA7\nwBbgOnd/PnKRJBIFGAnhIPBZ4FHg42a2re/159y9/3Ym0kAKMFKqzojRZzq/fqnz02sV3RepNRRg\npFSeTazqvxeStJQCjETTGWWa7fxY53Yf7u7H45ZMyqIAIzHdADzS8/v7wGFgY5TSSOl0qYCIBKOZ\nvCISjAKMiASjACMiwSjAiEgwCjAiEowCjIgEowAjIsH8H9ZVVd7SQ6Y9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7ab0a8eb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot\n",
    "fig = plt.figure(figsize=(4,3))\n",
    "plt.plot(encoded_output[:,0], encoded_output[:, 1], \"b.\")\n",
    "plt.xlabel(\"$z_1$\", fontsize=18)\n",
    "plt.ylabel(\"$z_2$\", fontsize=18, rotation=0)\n",
    "save_fig(\"linear_autoencoder_pca_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Stacked Autoencoders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /tmp/data/train-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# mnist dataset\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"/tmp/data/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Train all layers at once"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "n_inputs = 28*28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # encoding\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 1e-4\n",
    "\n",
    "dense_layer = partial(tf.layers.dense, activation=tf.nn.elu, \n",
    "                     kernel_initializer=tf.contrib.layers.variance_scaling_initializer(),\n",
    "                     kernel_regularizer=tf.contrib.layers.l2_regularizer(l2_reg))\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "h1 = dense_layer(X, n_hidden1)\n",
    "h2 = dense_layer(h1, n_hidden2)\n",
    "h3 = dense_layer(h2, n_hidden3)\n",
    "outputs = dense_layer(h3, n_outputs, activation=None)\n",
    "\n",
    "mse_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "reg_loss = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "loss = tf.add_n([mse_loss] + reg_loss)\n",
    "\n",
    "train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 mse: 0.0240489\n",
      "Epoch 1 mse: 0.0128124\n",
      "Epoch 2 mse: 0.0118014\n",
      "Epoch 3 mse: 0.0113695\n",
      "Epoch 4 mse: 0.0111495\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 128\n",
    "num_batch = mnist.train.num_examples // batch_size\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(n_epochs):\n",
    "        for t in range(num_batch):\n",
    "            X_batch, _ = mnist.train.next_batch(batch_size)\n",
    "            _, mse_train = sess.run([train_op, mse_loss], feed_dict={X: X_batch})\n",
    "        print(\"Epoch\", epoch, \"mse:\", mse_train)\n",
    "    saver.save(sess, \"./tmp/my_model_all_layers.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def show_reconstructed_digits(X, outputs, model_path = None, n_test_digits = 2):\n",
    "    with tf.Session() as sess:\n",
    "        if model_path:\n",
    "            saver.restore(sess, model_path)\n",
    "        X_test = mnist.test.images[:n_test_digits]\n",
    "        outputs_val = outputs.eval(feed_dict={X: X_test})\n",
    "\n",
    "    fig = plt.figure(figsize=(8, 3 * n_test_digits))\n",
    "    for digit_index in range(n_test_digits):\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 1)\n",
    "        plot_image(X_test[digit_index])\n",
    "        plt.subplot(n_test_digits, 2, digit_index * 2 + 2)\n",
    "        plot_image(outputs_val[digit_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./tmp/my_model_all_layers.ckpt\n",
      "Saving figure reconstruction_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeQAAAGoCAYAAACXNJbuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHY9JREFUeJzt3WuMnVXZP+BV2ulMW3pggEJbpRwKUkAOauWYQpSQGKJE\nDR8UEwmYSEw0SqIGEhP1ix+VmBA0ElAwETVBQzwgohjEeqDIqQQsh9ZaaKEUWjqdTjsd3q++//+6\n19tnz3Tmnva6Pt5P91rP3ruzf3mSe6014+233y4AwNQ6YqpvAAAQyACQgkAGgAQEMgAkIJABIAGB\nDAAJCGQASEAgA0ACAhkAEpg1RfPaHozpbsZU38DhYvPmzRP2exHtTDhjxsR9na3dDydynq7z9zL3\nZHxe000v3++yZcsO6APzhAwACQhkAEhAIANAAgIZABKYqqYugHGZ6uapyZi7l+Nxo/l7Gavraybr\nO5mMzzia42D+3/KEDAAJCGQASEAgA0ACAhkAEhDIAJCAQAaABCx7AqalrPtP9zLW2NhYpzla80Rz\nHHFE9+evrnNM5Htvfe5dP6/WWJn25faEDAAJCGQASEAgA0ACAhkAEhDIAJCALmuAhq6dxr2MNTo6\n2unfl1LKzJkzO91XX19ftT4yMhLOsW/fvvBaTauTeyIPqojmiV6T9SCS/5cnZABIQCADQAICGQAS\nEMgAkIBABoAEdFkDNERduBO5L/WsWfWf4j179oSvibqmo+7rqGO61Uk9NDRUrUed2f39/eFYg4OD\n1Xov3d9RV3r03lt62Zf7YPGEDAAJCGQASEAgA0ACAhkAEhDIAJCAQAaABCx7Ag4bk7HEpXVYwf79\n+6v16LCEaElQ69qcOXOq9Wg5Vmup0uzZs6v1aKlU9P5KKWVgYKBaj+639d6j5WDRe+xlOVQ0VmtZ\n23gPqvCEDAAJCGQASEAgA0ACAhkAEhDIAJCALmvgsBF1U7e6rKODDKJu29ZhDbt3767Wo0Mcjjrq\nqHCsqAs56nSOupyjeilx9/fevXs71Vt66TCPDuOIDqSI5iilfYBHTauTerzd+p6QASABgQwACQhk\nAEhAIANAAgIZABLQZf1f/vrXv1brt9xyS/iaZcuWVevR/qyf/vSnw7EGBwc71YG6rt3UUXduKaXs\n2LGjWn/jjTeq9Q0bNoRjvfLKK9X6zp07q/XjjjsuHGvBggXVetQ1feyxx1brxxxzTDhH9Lls3769\nWm/t8zxv3rxqPbrfuXPnhmNF9xx9Jq3O90gvHfnj5QkZABIQyACQgEAGgAQEMgAkIJABIAGBDAAJ\nzDiYLdwNUzLp/+Vd73pXtb5+/fpJmX/hwoXV+gUXXDAp8x9sJ554YnjtpptuqtZPOOGEg3Q34xbv\nMM+E2rx5c/X3ovXbFV2LluVEhz6UUsqWLVuq9WeeeaZaf/TRR8OxomVP0aEMrSWP0aEXr732WrXe\n399frUdLhUqJ3/tbb71VrbcOw4iWMUUHRZx00knhWKtXr67WV61aVa1HS65KKWV4eLhanzlzZvia\nSHTwxDve8Y4D+r3whAwACQhkAEhAIANAAgIZABIQyACQgMMl/ssvfvGLav3xxx8PX3PmmWdW6+vW\nravW//a3v4Vj/fKXv6zW77///mo96kJ86aWXwjm6ijogSyllyZIl1fqmTZs6zxN1YH/1q1/tPBaH\nlqhjOupoLSU+FGH//v3VetRp27oWjdXqWo7uOTqMpvX398ILL1TrW7durdajFQsbN24M54gOkYg6\nkE899dRwrKgr/MEHH6zWW79j0e/FypUrq/X58+eHY0XvJfp/d8QRB+851hMyACQgkAEgAYEMAAkI\nZABIQCADQAL2sk5kz5491fqGDRuq9ajL+sUXX5yoWyqzZ88Or0Vd1tF9RXvsllLKvffeW61fddVV\njbubUvayniTRXtYtUZd1tGd0q8s6es2+ffs6zV1K3GUd7WPfWrEQXRsYGKjWo32mo9+XUuLfpKgr\nPDoPoJRS/vjHP1br3/3ud6v1aAVLKaV88YtfrNYvuuiiar3VZR11f/fS3R9dW7Zsmb2sAWC6EMgA\nkIBABoAEBDIAJCCQASABgQwACThcIpFoucLpp5/eaZxog/WJFh2UsW3btmr9/PPPD8e64oorJuSe\noJT4wICofuSRR4ZjRUtZor/X1rKYvr6+aj06RGJwcDAc65xzzqnWo8MPorlPO+20cI7ovSxdurRa\nj5aClRIfnhPd14oVK8KxooMyouVYre8k+rzGxsbC10TGu4zYEzIAJCCQASABgQwACQhkAEhAIANA\nArqsaRoaGgqvffSjH63Wo+7E73znO+FYUXck9CLqpo4OS4k6pltjRd3B+/fvD8eKXhN1J8+dO7fz\nWJHovlp/e4sWLarWo4Mq1qxZE4715JNPVuvHHntstX7WWWeFY0UH20Td6tEBIS2tzuyDxRMyACQg\nkAEgAYEMAAkIZABIQCADQAK6rGm68847w2tbtmyp1o8++uhqffny5RNxSxxmov2Be+mCjfYtbu1B\nHHVZR/NH/76UuJs66gJu7Q0d3XPUTb179+5O/76UeMXE448/Xq3feuut4VhPPfVUtX7ppZdW66tX\nrw7HWrBgQbUeffYjIyPhWJHW93iweEIGgAQEMgAkIJABIAGBDAAJCGQASECXNaWUUl544YVq/cYb\nb+w8VrSf7fHHH995LJjIPYUnsjM76txtdUbv2bOnWo86naO9mUuJ30vUsR2NFe3vXUopw8PD1foD\nDzxQrf/pT38Kx4r2xT7//POr9VNOOSUcK9rjO/p8W/t+R59XL9394/2/6gkZABIQyACQgEAGgAQE\nMgAkIJABIAGBDAAJWPZEKaWU++67r1pvLeG4+uqrq/WTTz55Qu4JehUtP2kdIhHpeihDtIymlPiw\nhmhZTnSIQinxEp/ob3bOnDnV+lFHHRXOsXXr1mr9+eef73RPpZRy+eWXV+uXXXZZtd5a8hV9xtHn\n2zpAIzpEopeDSHr5//W/5hzXqwGACSGQASABgQwACQhkAEhAIANAArqsDzNRB+a9995brff394dj\nfetb36rWo65FmCxRt2vUOdsSdfRGBy/Mmzev8xxRl3WrY3vXrl3VevTeoy7rVgfyxo0bq/Woy3rp\n0qXhWKtXr67Wly9fXq23fnui37Hot6d1gEbrM65pdVI7XAIADgECGQASEMgAkIBABoAEBDIAJKDL\n+jBz++23V+sPP/xwtf7JT34yHMue1UylXrpdo9e09myPunCjOVpjDQwMVOtRl3Vrb+iu3dRR13LU\nSV1KKffcc0+1/uyzz1brH/7wh8OxLr744mr96KOPrtZb3c/RtWj/69a+2NFnPzo6Gr4mYi9rADgE\nCGQASEAgA0ACAhkAEhDIAJCAQAaABCx7OgQ9/vjj4bXPf/7z1fqiRYuq9W9+85sTck+QQbQspXXA\nQnRgQXRQxdy5czuPFS2VGhkZCceKDkyI/paHhoaq9V//+tfhHH/4wx86zX3OOeeEYw0ODlbrY2Nj\n1Xp0v6XES5Ki+4rmKCX+HqP/E+M9QKLFEzIAJCCQASABgQwACQhkAEhAIANAArqsp7Hh4eFq/ROf\n+ET4mqhz8JprrqnWHSBBVr10u7a6bSNRZ3RUbx0wEHVNR13Dre7v+fPnd5r/L3/5S7X+s5/9LJwj\nOngi+o05++yzw7G6dpi3vt/WYRFdRZ/9weymjnhCBoAEBDIAJCCQASABgQwACQhkAEhAl/U0EHWG\nXnnlldX6c889F461cuXKav0b3/hG9xuDaSbqQG51Rkedzn19fZ3H2rNnT6c5Fi5cGI41MDBQrUd/\n/3fffXe1vnbt2nCOD33oQ9X6tddeW62fcMIJ4VjRe4x+31pdztG1rvtSl1LK3r17O83Ry30dKE/I\nAJCAQAaABAQyACQgkAEgAYEMAAkIZABIwLKnaWD79u3V+kMPPdR5rLvuuqtaHxwc7DwWZNV1WUxL\ntIwpWsLUmiM6EGbOnDnV+ty5c8Oxtm7dWq0/8sgj1fpjjz1WrZ9xxhnhHNdff321ft5551Xr0WdS\nSnyIRLTsqXWARH9/f7UeLW9qfSfR/5Xoez+Yh054QgaABAQyACQgkAEgAYEMAAkIZABIQJd1Ijt2\n7KjWL7jggk7jRJvIlxJ3R8KhJOrcnTlzZuexog7dqNs2mruU+ECIaJVD61CETZs2Vetr1qzp9O+j\nTupS4g7sefPmVesjIyPhWNHnEn0mUb01Ty+HS0Sv6eXQi9bBIgfCEzIAJCCQASABgQwACQhkAEhA\nIANAArqsE7njjjuq9RdffLHTOJdcckl47WDuwwqTqdXR2nV/4l72Oo60Onpnz55dre/cubNa37Jl\nSzjWfffdV63/6le/atzd/2/hwoXhtWgv7Whf6paoazr6fFtzRN9j9JrW9zvezuj/Nt7fV0/IAJCA\nQAaABAQyACQgkAEgAYEMAAkIZABIwLKnSbZ+/frw2te//vXJuxGYJqJlKZO1hK/rIQP9/f3hWMPD\nw9X6m2++Wa2vXbs2HGvdunXVevR5nXrqqdX6SSedFM4RLReKlhe1DtaIDvaIPsfW8rFonui9t+5r\nIpc9jZcnZABIQCADQAICGQASEMgAkIBABoAEdFlPsocffji8Fm0wH1m5cmW1PmfOnE7jQGbT7UCU\nVtdu1Gk8MjLSeawlS5ZU69HhMieeeGK1fvzxx4dz7N69u1qP3kerM3p0dDS8VjOR33vrcInWtZpe\nDjU5UJ6QASABgQwACQhkAEhAIANAAgIZABLQZT0NXHTRRdX6Aw88UK3rsoaJ07VzttVNvHfv3k5z\ntDqgL7/88mo92hd76dKl1fqKFSvCOfr6+qr1rh3TLVGXcy97TE9GR/7BnMMTMgAkIJABIAGBDAAJ\nCGQASEAgA0ACAhkAEpjRS2v5BJiSSWECTa8TD6axzZs3T6vfi9aymLGxsWp9In+HoyVJ0fKi6KCI\n1rVe7jd6772YbgeOLFu27IBu2BMyACQgkAEgAYEMAAkIZABIQCADQAJT1WUNAPwXT8gAkIBABoAE\nBDIAJCCQASABgQwACQhkAEhAIANAAgIZABIQyACQgEAGgAQEMgAkIJABIAGBDAAJCGQASEAgA0AC\nAhkAEhDIAJCAQAaABAQyACQgkAEgAYEMAAkIZABIQCADQAICGQASEMgAkIBABoAEBDIAJCCQASAB\ngQwACQhkAEhAIANAAgIZABIQyACQgEAGgARmTdG8b0/RvDBRZkz1DRwuNm7c6PeCaW358uUH9Hvh\nCRkAEhDIAJCAQAaABAQyACQwVU1dAPwfZszo3jv49tsT1wMXzR/Vx8bGJmzuw5EnZABIQCADQAIC\nGQASEMgAkIBABoAEBDIAJGDZE8AUi5YqtZYwTdQSo76+vvBatLxpIpdW7du3r1qfOXNm+Jpo/iOO\n6P6M2cvSsoPFEzIAJCCQASABgQwACQhkAEhAIANAArqsJ9mPf/zj8NrQ0FC1vnbt2mr9+9//fuf5\nv/a1r1XrH/jAB6r1yy67rPMccCiJOnqjLuf9+/eHY42OjnZ6zfbt28Oxdu7cWa2PjIyEr+lyT6WU\nMmfOnGp90aJF1XqrYzvqmh4YGOhUb40VfSetju1MPCEDQAICGQASEMgAkIBABoAEBDIAJDBjIvck\n7WBKJp1Mn/vc56r1733ve5N8JwfmjDPOqNb//Oc/h69ZuHDhwbqd6SDPBriHuI0bN07p70XUAR11\nJw8PD4djbd68uVp/6qmnqvX169eHY7300kvV+gsvvFCtR13Zs2fPDuc47bTTqvVzzjmnWr/kkkvC\nsVauXFmtDw4OVuvz588Px4reS6TVSR5lYC97XEd7aS9fvvyABvOEDAAJCGQASEAgA0ACAhkAEhDI\nAJCAQAaABBwuMU6TsbzpvPPOq9Y//vGPV+utpRI//OEPq/VnnnmmWv/5z38ejnX99deH1+BQER1M\n8Oabb1br0XKkUuLlTX//+9+r9eeeey4cK1quc9JJJ1Xr0TKe3bt3h3Ns3bq1Wt+0aVO1/vrrr4dj\nRfNHS5Kiw3ZKKWXPnj3htZrW4RLRsrboNb0shzpQnpABIAGBDAAJCGQASEAgA0ACAhkAEtBlfQD+\n/e9/h9d+8IMfdBpr1apV4bXf/va31frcuXOr9WhT+KhrsJRSnn/++Wr9kUceqda3bdsWjgWHirGx\nsfDa3r17q/Vdu3ZV661O4+3bt1frRx55ZLW+evXqcKz3ve991fpFF11UrS9atKhajzq8Synl9ttv\nr9affvrpaj06dKKU+HcpOpBh3759nceKOs8nssv6YPKEDAAJCGQASEAgA0ACAhkAEhDIAJCALusD\n0Oo0jrr6om7q3//+9+FYUadlV3feeWd47R//+Eensa666qpx3g3k19qfOLoWdeHOmzcvHOud73xn\ntb5w4cJqPdrHvpRSPvjBD3Yaa+fOndV6tC91KfHe29FY0cqPUuLVItHnG3W3l1LKnDlzwmtdx4q+\nx6j7u9WRP16ekAEgAYEMAAkIZABIQCADQAICGQASEMgAkIBlTwfgPe95T3gtWhIVtf93bdfvRevA\ni1b7PxzqZs2q/+RFyxdLiZflDA4OVut9fX3hWEuXLq3WoyU2J554YjhW9JroAIvHHnusWr///vvD\nObZs2VKtv/vd767Wly1bFo4V/SZG72NkZCQcKxItrWr97kWHWHRdDjURPCEDQAICGQASEMgAkIBA\nBoAEBDIAJKDLepyijdwnw1133VWtP/HEE53HuuKKK6r1U045pfNYMNWiTtioczaqt0Rdw63fhKGh\noWo96sweGBgIx4q6qaPO6Ohgm6effjqcY/HixdX6pZdeWq2fffbZ4VhRJ/vu3bur9V27doVj9ff3\nV+t79uyp1lud0dF3v3///s5jjZcnZABIQCADQAICGQASEMgAkIBABoAEdFlPA//85z+r9c9+9rPV\nemsP2CVLllTrt9xyS7Xe2pcXppuJ7LKOOn1bf39RZ3ZUb+2x3XUv6w0bNlTr0V7dpZRy4YUXVutR\nN3WrwzzaMzraZ7r1nURnAoyOjnaau5T4M57Ifc8PlCdkAEhAIANAAgIZABIQyACQgEAGgAQEMgAk\nYNnTNLBmzZpqvbW8InLDDTdU66eddlrnsSCraGnK2NhYtR4tcSklXsoSzdH6uxweHq7W582bV61H\nhyWUUsrLL79crf/ud7+r1h999NFqfenSpeEc5557brW+atWqaj1avlVKfIjE3Llzq/VoaVMp8XcS\nfb+9LFWKXtPLErkD5QkZABIQyACQgEAGgAQEMgAkIJABIAFd1olcd9111fo999zTaZwvfelL4bWv\nfOUrncaC6SjqkI26lvfv3x+OFXXVRq9pdVlHncNDQ0PV+tatW8OxfvKTn1TrP/3pT6v1qJv49NNP\nD+c49dRTq/XFixdX663PMXrv0X21PsdW93lNL4d0jPegiF54QgaABAQyACQgkAEgAYEMAAkIZABI\nQJf1JNu1a1d47Te/+U21HnUUHnfccdX6zTffHM7R2msWDnW97D/d399frUcdxdHezK3X7Nu3r1p/\n8sknw7Eee+yxan3v3r3V+pVXXtmpXkop73//+6v1aM/olui9R2Pt2LEjHGtgYKBanz9/frU+Ojoa\njhV10euyBoDDlEAGgAQEMgAkIJABIAGBDAAJ6LKeZFdffXV47dVXX+001he+8IVqfXBwsNM4cLjo\npaM22oN5eHi4Wo/2Ri4l7oCO9qxet25dOFb0e7F8+fJq/eyzz67WV6xYEc4xa1Y9IrZt2xa+JhJ1\nn0cd7gsWLAjHilaeRN9Jq8s66vLu6+ur1qPPZCJ4QgaABAQyACQgkAEgAYEMAAkIZABIQCADQAKW\nPR0ka9eurdYfeuihzmN97GMfq9ZvvPHGzmPB4SxakhQdIFFKvFQpOiyhtSzmlVdeqdaj34XoAIlS\nStm5c2e1Hi1vWrhwYbUeLQUrJX7vQ0ND1Xp06EMppbz11lvVevR5RcvNSomXqUXLm6JDRUppv//J\n5gkZABIQyACQgEAGgAQEMgAkIJABIAFd1uMUbWZ+0003VetR12LLe9/73mp99uzZnceCw1nUndv6\nu4w6s6NDCVodvf/5z3+q9aib+oknngjHWrRoUbV++umnV+urVq2q1hcvXhzOER3iEB1g0/pNev31\n16v16DtpdVlHHe7RZx+9j1LiLuuoY7z1/bYOKTkQnpABIAGBDAAJCGQASEAgA0ACAhkAEtBlPU63\n3XZbtf7ggw92Huu6666r1u1ZDRMj6s5tdcdG16Ju2x07doRjbdiwoVrftGlTtX7kkUeGY0V73H/q\nU5+q1ufPn1+ttzqjo+7kBQsWVOvz5s0Lx+rr66vWo88x6mJvXXvjjTc6zVFKvI951F3fGmu8PCED\nQAICGQASEMgAkIBABoAEBDIAJCCQASABy57G6eabb56wsb797W9X6w6RgIMrWpJTSnz4QHSwzMaN\nG8Oxnn322U6vaR38cOaZZ1brraVSNa+++mp4bXR0tFqPDpeIPpOWaI7Wfe3cubNa37ZtW7XeWtYW\n/b5G9ej/w0TwhAwACQhkAEhAIANAAgIZABIQyACQgC7rRHbt2lWtR5ucT6Rog/VWR2G0Uf/IyEjn\n+aPuzFtuuaXzWJHovbQ65Vvdtxw6WisZor+/6P/Tyy+/HI61fv36av3111+v1hcuXBiOtWXLlmp9\n3bp14Wtqnn/++fBadLjEcccdV62ffPLJ4VivvPJKtT40NNTp37euRQdotO7r+OOPD69NNk/IAJCA\nQAaABAQyACQgkAEgAYEMAAnosk5k2bJlUzb3DTfcUK0vXbo0fE3U5XnrrbdOyD1Nltbn/pnPfGYS\n74SpMjY21vk1UZd1qzP/mGOOqdZnzar/FG/dujUc6+67767W16xZU61He0a3VkVEXd5HHXVU57Gi\n/aejemtf7Gi/7nPPPbdab32/b7/9dqd6a1/s8fKEDAAJCGQASEAgA0ACAhkAEhDIAJCAQAaABCx7\nGqdrrrmmWr/jjjsm+U7G57bbbjvoc0RLO0ppH2JRc+2114bXLrzwwk5jXXzxxZ3+PYee1rKYffv2\nVevRoROtgwyiZYQnnHBCtb5jx45wrOiAhX/961/Vei/LdaL3GL2PvXv3hmMtWrSoWt++fXu1vmDB\ngnCs6HCLaFlZ9O9LiQ8WiZY9HUyekAEgAYEMAAkIZABIQCADQAICGQASmDEVnWSllCmZdDL96Ec/\nqtZbXYhdPfHEE9X6RB7u8OUvfzm8tmLFik5jfeQjHwmvLV68uNNYCRy8Heb5XzZu3DhhvxdR13BL\ntDqgv7+/Wn/jjTfCsZ577rlq/cUXX6zWX3vttXCsV199tVp/6qmnqvWjjz66Wm91X+/Zs6dajw53\naHUzR59j1MUe3W8ppcyfP79aP+usszqPFR2gEXVfd10RUkopy5cvP6DfC0/IAJCAQAaABAQyACQg\nkAEgAYEMAAnosobe6LKeJL10WUe/a1FHcav7OnpNX19ftT46OhqOFe2ZHdWjDuRSStm/f3+1Pjw8\n3GmszZs3h3NElixZUq1H3c+ldL/fqMO7lFIGBgaq9Xnz5lXrUcd0KXHXdPR/opc9wXVZA8A0IpAB\nIAGBDAAJCGQASEAgA0ACAhkAEqjv9g0wjXVdmtJa/hmN1VqWE4mW+MyZM6dajw6wKCVeYtR1Gc+y\nZcvCOaKlXdEyotbysWhp19DQULXeWvIVjRV9V718v70sbxovT8gAkIBABoAEBDIAJCCQASABgQwA\nCeiyBg57rS7c6FCEqN7LgT1RR/HcuXPD10Rd3tFYe/furdZbndGRqMs56souJf68unZMl9L9nqei\nY7oXnpABIAGBDAAJCGQASEAgA0ACAhkAEtBlDdCDqHM36iYupZRZs+o/uVHXcGu/7KijeXR0tNMc\nra7waP5o7tZ779pl3cv+09OdJ2QASEAgA0ACAhkAEhDIAJCAQAaABAQyACRg2RNAD6JlRK0DFqLl\nOtHSn9YhCtEyoug10dy9HIbRy1KlrvMcqkubWjwhA0ACAhkAEhDIAJCAQAaABAQyACQwo5cOOwBg\nYnlCBoAEBDIAJCCQASABgQwACQhkAEhAIANAAgIZABIQyACQgEAGgAQEMgAkIJABIAGBDAAJCGQA\nSEAgA0ACAhkAEhDIAJCAQAaABAQyACQgkAEgAYEMAAkIZABIQCADQAICGQASEMgAkIBABoAEBDIA\nJPA/Qsg5S0GmIi4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7a81cd1d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./tmp/my_model_all_layers.ckpt\")\n",
    "save_fig(\"reconstruction_plot\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Tying weigts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "W1 = tf.Variable(initializer([n_inputs, n_hidden1]), dtype=tf.float32, name=\"W1\")\n",
    "b1 = tf.Variable(tf.zeros([n_hidden1,]), name=\"b1\")\n",
    "W2 = tf.Variable(initializer([n_hidden1, n_hidden2]), dtype=tf.float32, name=\"W2\")\n",
    "b2 = tf.Variable(tf.zeros([n_hidden2,]), name=\"b2\")\n",
    "W3 = tf.transpose(W2, name=\"W3\")\n",
    "b3 = tf.Variable(tf.zeros([n_hidden3,]), name=\"b3\")\n",
    "W4 = tf.transpose(W1, name=\"W4\")\n",
    "b4 = tf.Variable(tf.zeros([n_outputs,]), name=\"b4\")\n",
    "\n",
    "h1 = activation(tf.nn.xw_plus_b(X, W1, b1))\n",
    "h2 = activation(tf.nn.xw_plus_b(h1, W2, b2))\n",
    "h3 = activation(tf.nn.xw_plus_b(h2, W3, b3))\n",
    "outputs = tf.nn.xw_plus_b(h3, W4, b4)\n",
    "\n",
    "mse_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "reg_loss = regularizer(W1) + regularizer(W2)\n",
    "loss = mse_loss + reg_loss\n",
    "\n",
    "train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 mse: 0.0112707\n",
      "Epoch 1 mse: 0.00796578\n",
      "Epoch 2 mse: 0.00771752\n",
      "Epoch 3 mse: 0.00727718\n",
      "Epoch 4 mse: 0.00726337\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 5\n",
    "batch_size = 128\n",
    "num_batch = mnist.train.num_examples // batch_size\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(n_epochs):\n",
    "        for t in range(num_batch):\n",
    "            X_batch, _ = mnist.train.next_batch(batch_size)\n",
    "            _, mse_train = sess.run([train_op, mse_loss], feed_dict={X: X_batch})\n",
    "        print(\"Epoch\", epoch, \"mse:\", mse_train)\n",
    "    saver.save(sess, \"./tmp/my_model_all_layers_tying.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./tmp/my_model_all_layers_tying.ckpt\n",
      "Saving figure reconstruction_plot2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeQAAAGoCAYAAACXNJbuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHU1JREFUeJzt3VuMnVXZB/DVw3Rm6BTaKoVaUlBaoZAQDsrJKggJREkg\nYIgHYjBgIjHRRBI1kJioN14qXhg0GlA0EYMpRhNAwICggECBC6tQLCIKLfRMO9M5tHx336ef61n0\n3Z3OPNP+fpfPnr3Wu9/d7n/e5FlrzXrrrbcKADC9Zk/3BQAAAhkAUhDIAJCAQAaABAQyACQgkAEg\nAYEMAAkIZABIQCADQAJzp2le24Mx082a7gs4XGzfvt3vBTPawoUL9+v3whMyACQgkAEgAYEMAAkI\nZABIYLqaugB4G63jcWfNqvcJRfVorNYc0WuzZ9ef5aK5WxwB/H88IQNAAgIZABIQyACQgEAGgAQE\nMgAkIJABIAHLngAmUWsZT7RcqJclSV2XN3VdJlVK9+ttid6zb9++zmN1nWPOnDmTNsfB5AkZABIQ\nyACQgEAGgAQEMgAkIJABIAFd1gA9mMyu5b1793YeK+pOjjqNozn6+/vDOaL5u36O1lhRB/TY2Fg4\nVletLuuuXekHkydkAEhAIANAAgIZABIQyACQgEAGgAR0WQNMolZHb/Ra187olqg7OOrKnpiY6DxH\npHW9Xfeybl1XX19ftT53bj3Soq7w1vzTwRMyACQgkAEgAYEMAAkIZABIQCADQAICGQASsOwJOOy1\nDhKYzEMGxsfHO9VbS6iipT/RWNHynh07doRzRPNHy4iiayqllIGBgWp93rx51Xovy7GipVWt+xjd\nl+h7j+aYDJ6QASABgQwACQhkAEhAIANAAgIZABLQZQ0c9vbs2RO+FnUU79q1q9Pfl1LKyMhItT5/\n/vxqfWhoKBwrOkghGmvRokXVetSVXcrkdiBH9yX6HK0u6+j76nqARSlxB3bXz96af395QgaABAQy\nACQgkAEgAYEMAAkIZABIQJf1v3n88cer9VtuuSV8z7Jly6r1wcHBav3aa68Nx1q8eHGnOlAXdbvu\n3bu3Wt++fXvnsV555ZVq/V//+lc41ssvv1ytH3XUUdV61BldSinvfve7O40V1aMu51LizuixsbFq\nvdWxHb0Wzd/aFzt6T9SV3trLumtn9mTubf7/eUIGgAQEMgAkIJABIAGBDAAJCGQASEAgA0ACsw50\nM+weTcukb+ekk06q1tevXz8l80fLEs4999wpmf9gO+GEE8LXbrrppmp9+fLlB+lqDtjBW/vAf9i+\nfXvn34toyUq0XKd1uMSmTZuq9bVr11brTz/9dDhWdCBFf39/tR4dFFFKKbt3767WoyU+0W9967NH\nS4+iOVrXOzAw0Om6lixZEo51zjnnVOurVq2q1ltLqFpLorqKPsvChQv36/fCEzIAJCCQASABgQwA\nCQhkAEhAIANAAg6X+Dd33313tf7ss8+G7zn11FOr9T//+c/V+hNPPBGO9atf/apav++++6r1aHP5\nl156KZyjq9bG80uXLq3Wo033W6IO7K9+9audx4Koczb697xgwYJwrKgzO/r/NzEx8TZX99+i7tyo\nw7uU+DMODw93qkcHbpRSyrZt26r16J70sioiOtRn5cqV4XtWrFhRrUcHP8ybNy8cK+rIn6y/78IT\nMgAkIJABIAGBDAAJCGQASEAgA0AC9rJOJNpT9u9//3u1HnV5btiwYbIuqdmdGHVZR9f1xhtvhGOt\nWbOmWr/iiisaVzet7GU9RXrZyzrqtp09u/4M0vodjLpqd+zY0fWywv9P4+Pj1frIyEg4VrSXddRN\nHe3nHN2TUuIu79HR0Wr9xBNPDMeKVp5897vfrdaPO+64cKzrr7++Wl+9enW13vqM0WtRt3wvXdb2\nsgaAGUQgA0ACAhkAEhDIAJCAQAaABAQyACTgcIlEBgYGqvWTTz650zirVq2ajMt5W9FBGZs3b67W\nzznnnHCsSy65ZFKuCVqi5U2tZU/Ra+94xzs6zx8te4oOeGgtsYmWSnWdO1oiVkp8wEN/f3+nuUsp\nZd26ddV6tByrdVDFsmXLqvXoM7buY/TawTxEIuIJGQASEMgAkIBABoAEBDIAJCCQASABXdY0RRvY\nl1LKlVdeWa1H3Ynf+c53wrEGBwe7XRj0oJcu67lz6z+TUWd0q/s5ek90kEHrcJeoO3lsbKxaj/5f\nRtdUSvzZo/o///nPcKyHH364Wo8+4/ve975wrGOPPbZaj+596/ttdZl3/fsDPazJEzIAJCCQASAB\ngQwACQhkAEhAIANAArqsabr99tvD1zZu3FitR3v8Hn/88ZNxSfC2oo7irh21rfdEHb2tOfbs2VOt\n97KX9Zw5czrVo7GiruxS4g7oTZs2Ves///nPw7EeffTRan316tXV+tlnnx2ONTQ0VK1Hn6X1nUT3\nK3KgndQtnpABIAGBDAAJCGQASEAgA0ACAhkAEtBlTSmllL/97W/V+o033th5rMcee6xaj/afhakS\ndcjOnh0/m0T7TPcyVn9/f7UedWy3xor2so7eE80RjdMa65VXXqnWo/2qS4k7oy+44IJq/eijjw7H\niu59a1/uSC+d9weLJ2QASEAgA0ACAhkAEhDIAJCAQAaABAQyACRg2ROllFJ+/etfV+vRUolSSrn6\n6qur9fe85z2Tck3Qq2gpSy8HA0T/B+bOrf98RkubWtcVLS9qLeOJPsvo6GinuQcGBsI5RkZGqvUn\nnniiWt+yZUs41sUXX1ytn3POOZ2vK/qM0X1sHdKRiSdkAEhAIANAAgIZABIQyACQgEAGgAR0WR9m\noo7RNWvWVOutjtFvfetb1fqcOXO6XxhMgajTuNXNHHXuRvXoMIpS4s7srgdYtOaPOpAHBwc7zxEd\nIvHss89W68cff3w41kc/+tFqfenSpeF7uop+e+bNmxe+Z2xsrNMcrQM/DpQnZABIQCADQAICGQAS\nEMgAkIBABoAEdFkfZn70ox9V64888ki1/qlPfSocy57VHCpancZRZ3TUmT08PByOFXU6R93ffX19\n4VjRPNH1RntD79y5M5zjN7/5TbW+fv36an316tXhWFEHdrTyI9pHu5T4s0T3sTVW9N1H9/Fg8oQM\nAAkIZABIQCADQAICGQASEMgAkIBABoAELHs6BEUbv5dSyhe+8IVqfeHChdX6N7/5zUm5Jshg3759\n1XrrcImuh6VES29a80eHuPRyXdFBCtHyorVr14ZzRK8dffTR1fqll14ajrV48eJO19USLUmK7n1r\nWVsmnpABIAGBDAAJCGQASEAgA0ACAhkAEtBlPYNFG6Z/8pOfDN8TdW1ec8011boDJDiUzJ5dfwbp\npTM66vQdGhoKx4q6faPranUHR93UUf2ll16q1tesWRPOsXHjxmr9yiuvrNZXrVoVjhXd4+g3Keo8\nLyXuMI/Gan2/0Xuie98aK/oe95cnZABIQCADQAICGQASEMgAkIBABoAEdFnPAFGX52WXXVatP//8\n8+FYURfkN77xje4XBkl13dO41R3bdS/rVmd01Jk9MTFRrbc6eqPr2rp1a7V+5513Vuv3339/OMe5\n555brX/4wx+u1hcsWBCOFX2WqB7dq1JKGRsbq9anYs/qA+2kbo590EYGAPabQAaABAQyACQgkAEg\nAYEMAAkIZABIwLKnGSBaxvDQQw91HuuOO+6o1hcvXtx5LJhOrSVBXZc99XK4RLTsqLX0Znh4uNP8\n0UERLS+88EK1/sgjj1Trxx9/fDjWpz/96Wp95cqV1Xq0HKmlr6+vWo/ueyndD+PoZazpkOdKAOAw\nJpABIAGBDAAJCGQASEAgA0ACuqwT2bFjR7UebfAe+elPfxq+dsYZZ3QaC7Lq5SCB6MCCVndwdPDD\n+Ph4td7q2O7v76/Wux5gUUopr732WrV+9913V+s7d+6s1j/xiU+Ec5x33nnVetT9vWfPnnCsgYGB\naj36Hlv3JOqa7uXfRNeO/IPJEzIAJCCQASABgQwACQhkAEhAIANAArqsE7ntttuq9Q0bNnQaZ/Xq\n1eFrrQ5QOFREHbJRx3Tr/0XUmR2NNTg4+DZX99+i/ZSjzuhSSnn44Yer9QceeKBaX7ZsWbV+9tln\nh3NEnyW6v1EndSnxPY4+e2uP6Wj+vXv3TtpY08ETMgAkIJABIAGBDAAJCGQASEAgA0ACAhkAErDs\naYqtX78+fO3rX//61F0I8L9ay2KiQySiww+i5VClxEt/osMtnn/++XCsX/7yl9X6unXrqvVjjz22\nWj/iiCPCOboerBHVS4kP1ujlcIfocInW9zgTzOyrB4BDhEAGgAQEMgAkIJABIAGBDAAJ6LKeYo88\n8kj4Wmsj+ZpVq1ZV671sbg+Hg14OEoi6qaOxog7g1lijo6PV+saNG8OxFi9eXK1feOGF1foFF1xQ\nrS9YsCCcY3h4uFrv6+ur1qPPV0r7vnTVtTM70wESLZ6QASABgQwACQhkAEhAIANAAgIZABLQZT0D\nnH/++dX6/fffX63rsoa6qDu3l/dE9VY38d69e6v1aM/opUuXhmNF3dRRB/Tpp59erQ8NDYVzRKLP\n0YvofrW+q5nSNd2VJ2QASEAgA0ACAhkAEhDIAJCAQAaABAQyACQwa5raxw/NnnUOJ93Xz9CT7du3\nz6jfi16W6/SyjKjrYQ2zZ9efv+bOjVe/dl2S1MuyssihtLRp4cKF+3VjPCEDQAICGQASEMgAkIBA\nBoAEBDIAJDBdXdYAwL/xhAwACQhkAEhAIANAAgIZABIQyACQgEAGgAQEMgAkIJABIAGBDAAJCGQA\nSEAgA0ACAhkAEhDIAJCAQAaABAQyACQgkAEgAYEMAAkIZABIQCADQAICGQASEMgAkIBABoAEBDIA\nJCCQASABgQwACQhkAEhAIANAAgIZABIQyACQgEAGgAQEMgAkIJABIAGBDAAJCGQASGDuNM371jTN\nC5Nl1nRfwOFi27Ztfi+Y0RYtWrRfvxeekAEgAYEMAAkIZABIQCADQALT1dQFcNiZNatbL+Bbb8X9\nbNFY0Xtmz64/f7Xm6Krr5yullL1791br0fWWMrnXnIknZABIQCADQAICGQASEMgAkIBABoAEBDIA\nJGDZE8Akai3J6br0aHx8PBwrWmIULSOaM2dOtd7X1xfOERkZGel0Ta15JiYmqvV58+aFY+3bt69a\nby2Vmglm9tUDwCFCIANAAgIZABIQyACQgEAGgAR0WU+xn/3sZ+Fru3fvrtaffvrpav0HP/hB5/m/\n9rWvVesXXXRRtX7hhRd2ngMOB1FH8djYWPie0dHRan3Xrl3V+o4dO8KxhoeHq/VXX321Wt+6dWu1\nvmjRos5zrFixolqfP39+ONZRRx3V6T3R3KXEHdhR9/XcuXHURR3uvRyUcaA8IQNAAgIZABIQyACQ\ngEAGgAQEMgAkMKu17+pBNC2TTqXPf/7z1fr3v//9Kb6S/XPKKadU648++mj4nqhr8jAx9S2Yh6lt\n27Yd9N+LVkdttDd01NEb7fNcSimbN2+u1h966KFq/cknnwzH+uMf/1itv/nmm9V6dL1R53cppSxf\nvrxav+SSS6r1j3/84+FY0W/M0NBQtR6tOikl/k6ifbEHBgbCsSZTlKeLFi3ar98LT8gAkIBABoAE\nBDIAJCCQASABgQwACQhkAEjA4RIHaCqWN51xxhnV+sc+9rFqff369eFYP/7xj6v1devWVet33XVX\nONb1118fvgYzSWv55/j4eKexWsueHn/88Wr9nnvuqdajg2VKiZcdXnrppdV6tOyodYjDc889V62/\n8sor1fo//vGPcKyVK1dW69G9jw6QKKWUbdu2VeuzZ9efMfv6+sKxouVgkWhp1WTwhAwACQhkAEhA\nIANAAgIZABIQyACQgC7r/dDqHPzhD3/Yaaz3v//94Wv33ntvtX7EEUdU61EXYrTxeimlvPjii9X6\nH/7wh2o92gwfMosOi4g6eluHS8yZM6da37NnT7W+ZcuWcKzHHnusWn/jjTeq9ahjupRSrrnmmmp9\n9erV1frcufWf+z/96U/hHM8880y1Hq3KeO2118KxxsbGqvWoK731O9bf31+tR99Vq1M+ui+t+Q8W\nT8gAkIBABoAEBDIAJCCQASABgQwACeiy3g+tTuOoazPqpn7ggQfCsYaGhrpdWOD2228PX3vyySc7\njXXFFVcc4NXA1OvaTd3azzjaHzmq7969Oxwr2n/6qquuqtZbXdannnpqtT4wMFCt79q1q1qP9qUu\npZTt27dX69Fv1ZFHHhmO1Xqtpuse06XE30mrYzrq/o66r1ta3fr7wxMyACQgkAEgAYEMAAkIZABI\nQCADQAICGQASsOxpP5x55pnha9GSqOjgh8HBwUm5ppbWgRdRiz8cSqJDBiJ9fX3ha9Hym2iJzbHH\nHhuOFR0Iccwxx1Tr73znO8OxoqU80fKmV199tVpfu3ZtOEf0ntNOO61aX7p0aThWtBQtMjExEb4W\nLe2KvpPh4eFwrGh5UzR/9NteSvfP+P95QgaABAQyACQgkAEgAYEMAAkIZABIQJf1AYo2i58Kd9xx\nR7X+3HPPdR7rkksuqdZPPPHEzmPBVOhlI/+oC7fVlR111b755pvVeqszetmyZdV61OXd6jSOPn/0\nnqeeeqpa/+1vfxvOsXXr1mp9yZIl1XqryzoSHcbR6mY+4ogjqvXx8fFqvb+/Pxyr6yEWrb93uAQA\nHAIEMgAkIJABIAGBDAAJCGQASECX9QzwzDPPVOuf+9znqvXR0dFwrKgL8pZbbqnWW3v8wnRq7Rvc\ndf/pXkRdwL38n4m6g1vXG83zwgsvVOv33XdftR51i5dSyoc+9KFq/eKLL67WTzrppHCsaO/t6Pdq\n/vz54Vhd9+SP7m8p8X2M7v2B7lfd4gkZABIQyACQgEAGgAQEMgAkIJABIAGBDAAJWPY0Azz22GPV\nemt5U+SGG26o1t/73vd2Hguy6ro0pXWIQ7SEKjqQovX/MlquE13vggULwrG2bNlSrd9zzz3V+rPP\nPlutn3LKKeEcl19+ebV+/vnnV+utQzp27txZrUcHRbSWfEXfSTTH4OBgOFa07KnroROTwRMyACQg\nkAEgAYEMAAkIZABIQCADQAK6rBO57rrrqvU777yz0zhf+tKXwte+8pWvdBoLZqJZs2ZV61GXc+tA\niGis6MCC3bt3h2NF3dRz59Z/inft2hWOde+991brv/jFL8L31Hzwgx8MXzvrrLOq9agzeseOHeFY\n0X088sgjq/VWl3V0X6LvJDoIpHVd0XcV/X3rPfvLEzIAJCCQASABgQwACQhkAEhAIANAArqsp1ir\nazLag3bPnj3V+jHHHFOt33zzzeEcrW5DOFRE+xBHnbutfYujrtpoz+qoA7klmuOpp54K37NmzZpq\n/YUXXqjWr7322mr9ggsuCOdYvnx5td7f31+tt37fBgYGqvWoMznqiG+9FnXLR9fbmr9r9/Vk8IQM\nAAkIZABIQCADQAICGQASEMgAkIAu6yl29dVXh6+9/vrrncb64he/WK0vXry40zhwuIg6ZFt7WQ8P\nD1frUWd2ayVDtGf15s2bq/UHH3wwHOvxxx+v1k8++eRq/SMf+Uinvy+llIULF1brUZfz4OBgOFbU\n6Rx9J1u2bAnHiu79nDlzqvXW/tOt17r+vb2sAeAQIJABIAGBDAAJCGQASEAgA0ACAhkAErDs6SB5\n+umnq/WHHnqo81hXXXVVtX7jjTd2HgsOB9HSlKi+d+/ecKxoKU106EtrWUw0z1//+tdq/f777+98\nXdFhESeccEK1PjExEc4RfcboEInowI1S4iVU0Xta1xUd4BHdk8k8EMKyJwA4xAlkAEhAIANAAgIZ\nABIQyACQgC7rAzQyMlKt33TTTdV6tCl7y1lnnVWttzaxh8NZ1AkbdcG2uoMjAwMDnceKDpC56667\nqvUNGzaEY5100knV+ooVK6r1BQsWVOutDvOdO3dW69HvXtSVXUp8sEYk6soupZShoaFqPfosrc/4\n5ptvVuvRgSNdP0cXnpABIAGBDAAJCGQASEAgA0ACAhkAEtBlfYBuvfXWav3BBx/sPNZ1111Xrduz\nGv5ba0/hSNRl3erC7dqdvHXr1nCs3//+99X67373u2q91Wl80UUXVeuXXXZZtb58+fJqvbXyI+pA\nju5J63pnz64//0Wd2YODg+FY0Z7VUVd4tPd267qif1/R3K2x9pcnZABIQCADQAICGQASEMgAkIBA\nBoAEBDIAJGDZ0wG6+eabJ22sb3/729W6QyTgv0VLmFqipSwTExPhe6KlLNHyl+3bt4djPfXUU53e\nc9ppp4VjnXnmmdX6McccE76nq+i3JzpYo7XsJ7r3XQ8CKaWU1157rVrfuHFjtb5p06ZwrBNOOKFa\njw6waH3GXv5N/sfYB/RuAGBSCGQASEAgA0ACAhkAEhDIAJCALutEog3QD3TD8v3R399frbc2Uo82\n1x8dHe08/8jISLV+yy23dB4rEn2WVqd8X1/fpM3P1Ona0Rt11JYSd85GYw0PD4djbdmypVqPDnFo\ndX/v3r27Wt+wYUO1Hv1/jQ53KKWUffv2VetLliyp1ufOjSMlmmfz5s3VenRQRCml/OUvf6nWo/u4\ncuXKcKzo93V8fLxab/0m9nLgyX9cywG9GwCYFAIZABIQyACQgEAGgAQEMgAkoMs6kWXLlk3b3Dfc\ncEO1/q53vSt8T7Rv7Pe+971Juaap0rrvn/3sZ6fwSpgsUWd01DXcy0qGaI5WZ37U6RytMnjmmWfC\nsaKO3gULFnSau9UZHY111FFHdbqmUuL9uqN73+pWf/HFF6v1U089tVo/8cQTw7GifxOR1n7VuqwB\n4BAgkAEgAYEMAAkIZABIQCADQAICGQASsOzpAF1zzTXV+m233TbFV3Jgbr311oM+R2t5RWvD9prP\nfOYz4WvnnXdep7E+8IEPdPp7Zq5oiU1ruUrrgIeahQsXhq+ddtpp1fq6deuq9U2bNoVjPffcc9V6\n9Bmjwyhah0vMnz+/Wl+8eHG13lqqdNxxx1XrRx99dLX++uuvh2Odfvrp1fqKFSuq9cHBwXCsaNlT\n10NFJoMnZABIQCADQAICGQASEMgAkIBABoAEZrU2yj6IpmXSqfSTn/ykWh8bG5u0OaIuy8k83OHL\nX/5y+FrU0Ri5/PLLw9eWLFnSaawEDl6rJf9h27ZtnX8vot+1qNO/dcBA9FrX7txSStmyZUu1/vLL\nL1fra9euDceKur+j69qxY0e13upmjn6vovvY6mY+5ZRTqvV58+ZV60uXLg3HGh8f7/Se6JCMUuIu\n7/7+/mo9ut6WRYsW7dfvhSdkAEhAIANAAgIZABIQyACQgEAGgAR0WUNvdFlPkV66rLtq/Q5G+6zv\n3bu38zytbt+a0dHR8LVoz+qoy3rnzp3V+rZt28I5or2pjzzyyGp9aGgoHCvqzI4+R9RJXUr3zvfW\nPvpR13T0vbf2so7+HemyBoAZRCADQAICGQASEMgAkIBABoAEBDIAJGDZE/TGsqcpMpnLnqLfu2jp\nTSnt5Tddx4qW5XSdo5R4WU7rgIeuf9/X11etR9e7Z8+ecKzovkTLx1rZFL2ntSQp0vXfRC+ZadkT\nAMwgAhkAEhDIAJCAQAaABAQyACQQ77gNQNjNHHUaR53UpZQyMDDQaY7du3eHY0VdwNGBEGNjY9V6\n6wCLqMu6v7+/Wm91IEf3q+v9LSU+LCKav5fO6OlYgeQJGQASEMgAkIBABoAEBDIAJCCQASABXdbA\nYSPa67jVURt1M0edvq39lKN5oq7lqAO5lFLmzZsXvlYzMjLS6e9Libuse+lAju5j1JXey57gvehl\n/+uDxRMyACQgkAEgAYEMAAkIZABIQCADQAICGQASsOwJoCFa4hMty2kto4mW60QHPLSW/kTLmKKl\nUlG9lwMhelkqNJkHP0R6WdaWiSdkAEhAIANAAgIZABIQyACQgEAGgARmzZTuMwA4lHlCBoAEBDIA\nJCCQASABgQwACQhkAEhAIANAAgIZABIQyACQgEAGgAQEMgAkIJABIAGBDAAJCGQASEAgA0ACAhkA\nEhDIAJCAQAaABAQyACQgkAEgAYEMAAkIZABIQCADQAICGQASEMgAkIBABoAEBDIAJPA/3X0+vVNS\nuygAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7a7c40cf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./tmp/my_model_all_layers_tying.ckpt\")\n",
    "save_fig(\"reconstruction_plot2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Training one Autoencoder at a time in multiple graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# a graph for a layer\n",
    "def train_autoencoder_layer(inpt, n_units, n_epochs, batch_size, learning_rate = 0.01, \n",
    "                            l2_reg = 0.0005, activation=tf.nn.elu, initializer=tf.contrib.layers.variance_scaling_initializer()):\n",
    "    graph = tf.Graph()\n",
    "    with graph.as_default():\n",
    "        n_inputs = inpt.shape[1]\n",
    "        regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "        X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "        \n",
    "        h = tf.layers.dense(X, n_units, activation=activation, name=\"hidden\", kernel_initializer=initializer,\n",
    "                           kernel_regularizer=regularizer)\n",
    "        outputs = tf.layers.dense(h, n_inputs, activation=None, name=\"outputs\", kernel_initializer=initializer,\n",
    "                           kernel_regularizer=regularizer)\n",
    "        \n",
    "        mse_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "        reg_loss = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)\n",
    "        loss = tf.add_n([mse_loss] + reg_loss)\n",
    "        \n",
    "        train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "        init = tf.global_variables_initializer()\n",
    "    \n",
    "    with tf.Session(graph=graph) as sess:\n",
    "        num_batch = len(inpt) // batch_size\n",
    "        sess.run(init)\n",
    "        for epoch in range(n_epochs):\n",
    "            for t in range(num_batch):\n",
    "                indices = np.random.permutation(len(inpt))[:batch_size]\n",
    "                X_batch = inpt[indices]\n",
    "                _, mse_train = sess.run([train_op, mse_loss], feed_dict={X: X_batch})\n",
    "            print(\"Epoch\", epoch, \"mse:\", mse_train)\n",
    "        params = dict([(var.name, var.eval()) for var in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)])\n",
    "        h_val = h.eval(feed_dict={X: inpt})\n",
    "        return h_val, params[\"hidden/kernel:0\"], params[\"hidden/bias:0\"], params[\"outputs/kernel:0\"], params[\"outputs/bias:0\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 mse: 0.0189829\n",
      "Epoch 1 mse: 0.0184789\n",
      "Epoch 2 mse: 0.0191758\n",
      "Epoch 3 mse: 0.0191016\n",
      "Epoch 0 mse: 0.00448945\n",
      "Epoch 1 mse: 0.00456623\n",
      "Epoch 2 mse: 0.00490648\n",
      "Epoch 3 mse: 0.00482273\n"
     ]
    }
   ],
   "source": [
    "hidden_output, W1, b1, W4, b4 = train_autoencoder_layer(mnist.train.images, n_units=300, n_epochs=4, batch_size=150)\n",
    "_, W2, b2, W3, b3 = train_autoencoder_layer(hidden_output, n_units=150, n_epochs=4, batch_size=150)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# stack \n",
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=[None, n_inputs])\n",
    "hidden1 = tf.nn.elu(tf.matmul(X, W1) + b1)\n",
    "hidden2 = tf.nn.elu(tf.matmul(hidden1, W2) + b2)\n",
    "hidden3 = tf.nn.elu(tf.matmul(hidden2, W3) + b3)\n",
    "outputs = tf.matmul(hidden3, W4) + b4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa8AAAFsCAYAAAB7FzYbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGxpJREFUeJzt3UtsVWX3x/EHSqEXSlt64VKRSysCFaOIUdREfQc6MNGo\ncYAONGqiMdFEEjWQmKgTnSkTo0bjfWBi4m2gRk1IEAsqKmKJiIWKGKC0QFt6oQV9J77/9/2zfo99\nds9pD6v9foYr+9l7n3M4Xdk5P9Yz5a+//goAAHgytdA3AABAVjQvAIA7NC8AgDs0LwCAOzQvAIA7\nNC8AgDs0LwCAOzQvAIA7NC8AgDs0LwCAO9MKdF1mUiHfphT6Biarzs5Ovs/Iq9ra2hG/zzx5AQDc\noXkBANyheQEA3CnUb14AMGamTBm/n0DVtdRWU1m2n0o952TGkxcAwB2aFwDAHZoXAMAdmhcAwB2a\nFwDAHdKGAFzLkiwcixTitGn2z+iff/5palOn6mcFdWxqLaaoqCj5+rncUyHx5AUAcIfmBQBwh+YF\nAHCH5gUAcIfABgA3VOAi1yBCllFOar0KRyixsEjq9YeHh+V6dX11nzNmzBjpFv9Pru/peODJCwDg\nDs0LAOAOzQsA4A7NCwDgDoENAGedWGBC1bNMg1Drp0+fbmpDQ0NyvQpdqGOLi4uTrh2Cvld1zljg\nQ4Urct37K8skEvVaT58+nfd7OhNPXgAAd2heAAB3aF4AAHdoXgAAd2heAAB3SBsCOOtkGaWUJW04\nODhoav39/aam9uiK3VdpaamppY6MCkGnBcvLy01NJfhi11KjoNRrD0G/p6dOnUq6zj/d15lin+lo\nx0vx5AUAcIfmBQBwh+YFAHCH5gUAcIfABoCzTmzvKhUkUMcODAzI9Wrskjo2tp9VTU2Nqakgg6qp\nYEgI6eEINcYqhPSRVbH3NEvgJXV9lvFSo8WTFwDAHZoXAMAdmhcAwB2aFwDAnUkZ2Ni6daupbdy4\nUR7b0NBgaup/1N95551y/ezZs5NqwGSgfshXExqyBC6OHTtmal1dXXJ9R0eHqfX09JhadXW1XK8C\nG2qaRVtbm6l1dnbKc6rXdPLkSVOrq6uT66uqqkxNTc0477zz5PqKigpTq6ysNLWysjK5XgVOSkpK\nTC3LfmQpePICALhD8wIAuEPzAgC4Q/MCALhD8wIAuDNFpVLGQUEu+h/nn3++qe3Zs2dMrqVSO5df\nfvmYXCvfFi1aJOvr1683tXPPPXeM72ZEYz+PBlJnZ2fy91mNElJpw9gopQMHDiTVfv/9d7n+0KFD\npqZSgLH9vNSIJZVsVPevrh1CerJPpZxD0GlHdeysWbPk+uuvv97UVq5caWozZ86U61WKUNVi+4Gp\nY2tra0f8PvPkBQBwh+YFAHCH5gUAcIfmBQBwZ1KOh3r//fdN7YcffpDHNjc3m1pra6upbdu2Ta7/\n4IMPTO3TTz81tcWLF5vavn375DlTxX50njdvnqnFfuBWVJDjscceS16PyUsFxFRgo7e3V65Xo5S6\nu7tNTY1XCkEHRtSxsZFFfX19pqZCHCqwERvvpAIXKgAVGy/122+/mZp6T5YsWZK8ftWqVUn3GYL+\nTMcDT14AAHdoXgAAd2heAAB3aF4AAHcmZWBj+fLlSbWYCy+80NTWrl0rj33mmWdMrb293dRUYGPv\n3r3J96RMnz5d1lVgQ13/yJEjcv2yZctyui9MfLEf8dU0hdQQRwg6HKH+nat9t0LQe08VFxebmtqP\nKgQdglKvaXBw0NTUJI0QdDhDBT6+++47uX7nzp2mtn//flNramqS69X7p97/2Geqwi3q2HwHO3jy\nAgC4Q/MCALhD8wIAuEPzAgC4Q/MCALgzKdOG40mlllLTelkSkFmoUVZq9Mxll10m11933XV5vydM\nDqlpQ7WfVQghVFdXJ50zth9YbI+6M8XShmrEk0orqvFMsf24FixYYGo9PT2mdvjwYbleXb+8vNzU\nqqqq5HqVglTnVO9zjPpMY+tHm0LkyQsA4A7NCwDgDs0LAOAOzQsA4A6BjQlM7T0UQgg333yzqal9\njp577jm5PvbDM/Afsf2w1I/zKhygaiGEUFFRYWpqn6nY9VPDCbG9q4qKimT9TOo+Y4EJNZ5JfXdV\niCOEEAYGBkxt7ty5pjZ//ny5XoVY1HgotZdaCPrvgfp7kmVkWAqevAAA7tC8AADu0LwAAO7QvAAA\n7hDYmMBee+01WT906JCpqf2PFi5cmO9bwiSRZWqCCkzMnDkz+bwq8BALfKjzqj261H5csbq6lgpn\nxN4TFc5obW01te+//16u7+3tNbXGxkZTW716tVyv9vJTgZVYiEVRgRkV4sgFT14AAHdoXgAAd2he\nAAB3aF4AAHcIbEwQbW1tprZu3brk9S0tLaam/pc+kG9qwkJs6kLqNIxYOEJNjlDUliYhhHDy5ElT\nU1M7VDghFlhQ2xFt2rTJ1LZv3558T83NzUm1EHS4JPV9ih2rXitbogAAJj2aFwDAHZoXAMAdmhcA\nwB2aFwDAHdKGE8RHH31kasPDw/LY2267zdSWLFmS93sCUqi0WWw/rrKyMlNTybby8nK5Xp1XJQtj\n46FSk3XqOrFz7tq1y9TUKKhYAnLevHmm1tDQYGpqBFwIejzWqVOn5LGKShGqz3S0+3bF8OQFAHCH\n5gUAcIfmBQBwh+YFAHCHwIZDKojx3nvvmVps/52nn37a1IqKinK/MSBPYqOUUveUUiOTQkgPJ8Su\nr0IP9fX1pqaCHe3t7fKcX375ZdKxTU1Ncv2ll15qaitXrjQ1NQYqBP2eqNrQ0JBcr15raogjFzx5\nAQDcoXkBANyheQEA3KF5AQDcIbDh0CuvvGJqmzdvNrXbb79drmeaBgpF/WivamrfrhB0WEkdGwsH\n9Pb2mpoKHFRUVMj1tbW1pqbCDcePHze1b7/9Vp7z559/Trqnq6++Wq5ftWqVqa1YscLUSkpK5PrU\nqSEx6r1W51TvUy548gIAuEPzAgC4Q/MCALhD8wIAuEPzAgC4Q9rwLPbDDz/I+oMPPmhqavTLU089\nlfd7AvJNJdtUWi2EeGLuTLG04vTp002tr68v6bgQQpg5c6apqQSj+u5+/vnn8pytra2mdu2115ra\nFVdcIdcvX77c1FQqMra/n3r/1Xit2GeiRsvF3r/U66fgyQsA4A7NCwDgDs0LAOAOzQsA4A6BjbPE\nwMCAqa1du1Yeq344veOOO0yNMVAoFLWfU6yeuh9UCHoUUWzvLkUFPkpLS5NqIei9v44cOWJqX3/9\ntam1tLTIc5aXl5vaeeedZ2qrV6+W6xsbG01NjWJSf2NCCGHqVPsMoz6TLCGMfO/dpfDkBQBwh+YF\nAHCH5gUAcIfmBQBwh8BGAaj/UX7DDTeY2u7du+V69T/qn3zyydxvDMiT2A/2qXs/xUIY6rujAgeq\nlmV9LDDS3t5ualu2bDG1bdu2mVpnZ6c855o1a0ytqanJ1BoaGuR6FUKJTdNQVJCjrKzM1GLvydDQ\nkKmpwEhsksZowx08eQEA3KF5AQDcoXkBANyheQEA3KF5AQDcIW1YAEePHjW1TZs2Ja9/8803TW32\n7Nm53BIwLlRiTe0HFUvLqb23ZsyYYWqxfb/U9VXarr+/X67/5ZdfTG3z5s2mtnXrVlOLJSDnz59v\naitWrDC1WbNmyfXqvCoBGEv1qfXq2Nh69fmp9zmWVhwtnrwAAO7QvAAA7tC8AADu0LwAAO4Q2Bhj\n3d3dpnb55ZcnrX3rrbdk/eKLL87pnoBCSQ1sxPaOKi4uNjUV7ojtXaWCHCqwcPjwYblejYdSx6rA\nxNKlS+U5V65caWo1NTXyWEW9VrXvmBrDFYJ+/bFRTqnU+tj1R4snLwCAOzQvAIA7NC8AgDs0LwCA\nOwQ2xtirr75qanv37k1ae9VVV8l6vv+nOjBe1JQGFdiISZ3moAILIYQwODhoaipIcOLECbm+qqoq\n6ZwXXXSRqa1atUqec9GiRaamXlMsRKGur95nFXYJIT7540yxwEWWe80nnrwAAO7QvAAA7tC8AADu\n0LwAAO7QvAAA7pA2zJM9e/bI+hNPPDG+NwI4o5JpsQSiStGp9bH9wFSyrqenx9Ta2trk+m3btpma\nSttVV1eb2ty5c+U5S0tLTU29ztjIKyXLyCf1XqUmEEPIlhbNJ568AADu0LwAAO7QvAAA7tC8AADu\nENjIk82bN8u6+jFYWb58uampH3KBySAWLlBBBiU2Cknts6XGK5WVlcn1c+bMSbonddySJUvkOVU4\nIktgQwVGpk2zf9pjY+WyhDMU9VmNxwg7nrwAAO7QvAAA7tC8AADu0LwAAO4Q2CiAK664wtQ+++wz\nUyOwAfx/KsigwgGxEIL6TtXV1SUdF0IICxYsMLWuri5TU4GRmpoaec7UaRixEEssnHKmWIgi9T3N\nIjVYkwuevAAA7tC8AADu0LwAAO7QvAAA7tC8AADuTBmPVIhQkItiQhv7eTSQOjs7x+37nPr3Kkta\nTqX9Tp8+nXxeNXIq9TqxeurIqNg95XLcP11rvNTW1o54szx5AQDcoXkBANyheQEA3KF5AQDcKVRg\nAwCAUePJCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0L\nAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDg\nDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAODOtAJd968CXRcT15RC38Bk1dHRwfcZ\neVVfXz/i95knLwCAOzQvAIA7NC8AgDuF+s0LAMbMlCn6J5O//srt5zm1Xl0rdv18X3sy48kLAOAO\nzQsA4A7NCwDgDs0LAOAOzQsA4A5pQwATTiyZp+qpCcIQQjh9+rSp/fnnn6ZWXFycfE51/WnT7J/m\n4eHh5PW5JiDHIi2Zbzx5AQDcoXkBANyheQEA3KF5AQDcIbCRJ2+//bas9/X1mdr27dtN7aWXXkq+\n1uOPP25q//rXv0ztmmuuST4n4EGuQQIVrlBBCPW9DSGEgYEBUzt16pSpTZ1qnwtmzJiRcoshBB3Y\niN2TCpFUV1cnX18FPoqKipLuKYTxG491Jp68AADu0LwAAO7QvAAA7tC8AADuTCnQHjGuN6Z54IEH\nTO3FF18swJ3814oVK0ztyy+/lMdWVlaO9e0Uwtk/EmCC6ujoyPv3OcsP/iqEoUIMIYTQ09NjaocP\nHza1X3/9Va4/ePCgqZ04cSLpnlQIIoQQZs6cmbR+zpw5cn1jY6OpzZ0719RKSkrkemX69OmmFrt/\nNU1EydJr6uvrR/wHwJMXAMAdmhcAwB2aFwDAHZoXAMAdmhcAwB3GQ41gLJKFF198sandeuutprZn\nzx65/vXXXze1Xbt2mdq7774r199zzz0j3SJQULFkmkoRqvFO/f39cv2OHTtMbefOnab2yy+/yPUq\nrVhaWmpqZWVlphZL5R06dMjU1GtqamqS69XYp4aGBlNTCcYQdLJQjbdStRD0Z6KSiVn2M0vBkxcA\nwB2aFwDAHZoXAMAdmhcAwB0CG3/bv3+/rL/88stJ6y+99FJZ/+STT0xN/ZirfjSNjbhRo2u2bNli\nap2dnXI9UCipY59i//aHhoZMTYUbjhw5ItcfO3bM1NR4J/V9DEGPYZs/f37S+qNHj8pz9vb2mpra\nI6y2tlaur6mpMTUVrigvL5fr1bHq+up9DkHv8xX7/FKvn7RuVKsAACggmhcAwB2aFwDAHZoXAMAd\nAht/i4Ub1P/+VuGMzz//XK5Xe/Wkeu2112T9m2++SVp/0003jfrawFhQ3ycV4ogFO9SUCvXdjU2T\nqK6uNjU1uaK5uVmuX7NmjanV1dWZmprE8eGHH8pzqgkZqrZgwQK5Xr0mFZhQk0BCiAcxzhQLsaj1\nWSZ0jBZPXgAAd2heAAB3aF4AAHdoXgAAd2heAAB3SBv+bdWqVbKukkwqdRNL8uQiNppKjcgBJpLY\nHk8q2aaSecePH5frZ82aZWpqvNPy5cvl+tRk3759+0ytra1NnlPt27d48WJTKykpketVim9wcNDU\nYn831CgolepUY6BC0MlOdc7Y+tHiyQsA4A7NCwDgDs0LAOAOzQsA4A6BjRFUVlaOy3XefPNNU9ux\nY0fy+uuuu87UGhsbc7onIEXqHl0xWcYGqWPV/nix760KgqjviQp2hKCDCCqc8cUXX5ia2ksshBDO\nPfdcU1u9erWpXXjhhXK92s/r5MmTphYLbBQVFZla6h5fIej3VJ0zFsIZ7b8fnrwAAO7QvAAA7tC8\nAADu0LwAAO4Q2CiA77//3tTuu+8+U1M/uoYQwrx580xt48aNpqb+lzxQSKn7eakf/EPQ0zSU2bNn\ny7qajqOOje0HduDAAVNT4YzYNA1l0aJFprZw4UJTU8GMEPR7paZZxIIR6rVmCdHEghip1x8tnrwA\nAO7QvAAA7tC8AADu0LwAAO7QvAAA7pA2LICWlhZTiyULlfvvv9/Uli5dmtM9AfmUZRSQOjaW9lPr\nVQIxlmxTY5/Uflzd3d1y/a5du0xNJQu7urpMLbYfl9pPbMGCBaZWVVUl1ysDAwOmFktwqr2/+vv7\nk9erBGdslFQ+8eQFAHCH5gUAcIfmBQBwh+YFAHCHwMYYu/vuu03tnXfeSVr78MMPy/qjjz6a0z0B\n+aQCF1kCGyqcoUIUIehwRl9fn6llGW+k1re2tspjv/rqK1P7/fffTU3tnaX2HQtBv34VjoiFWNS1\n1LFqZFQIOkhy4sQJeawyPDycdK3UMVKpePICALhD8wIAuEPzAgC4Q/MCALhDYCNPYj9wfvzxx6am\n/kf7nDlzTG3Dhg3ynOp/tANnkyx7N6lwQSycoMIBKnAQCweoyQ+9vb2m9uOPP8r1anKGCpGowEhd\nXZ08p9pPrKKiwtR6enrk+mPHjpmaev/r6+vl+tLSUlPL8p4WCk9eAAB3aF4AAHdoXgAAd2heAAB3\nCGzkyW233SbrHR0dSesfeughU1M/5AKeqR/9VbggFvhQkxvUNAoV7AhBbz2kpmns3r1brj948KCp\nqe+pClXFAhPnnHOOqanAh7p2CCEcP37c1IqLi00ttqWJui/1OcWmnqhrqWOzTD1JwZMXAMAdmhcA\nwB2aFwDAHZoXAMAdmhcAwB3ShqOwfft2U9u0aVPy+ltuucXU1q1bl8stAW6lJhBD0KOY1Min2N5V\nasSSSvH19/fL9eq8apTV4sWLTa2xsVGec+HChUnXib0mtU+YSjuqpGUIIXR3d5uaSgbGxtKpY2PJ\nxHziyQsA4A7NCwDgDs0LAOAOzQsA4A6BjREMDAyY2vr1601taGgo+ZyXXHKJqbFHFyaD1FFQWfaO\nUqOgYntftbW1mVp7e7up/fHHH3K92nevqqrK1ObNm2dqzc3N8pyzZs0yNRWuiIVIqqurTU3tB6bG\nSMXq6jOprKyU61VgRu0RFtujbbRjo3jyAgC4Q/MCALhD8wIAuEPzAgC4Q2BjBC+88IKpffHFF8nr\n7777blNjmgbwz9QkjRB0MEoFGWL76KnpOJ2dnaamghkhhLBs2TJTU9M0amtrTS02NUSFtY4dO2Zq\nKhgSq6sQRW9vr1y/d+9eUysvLze1pqYmuV6FQ1SIJrafWKw+Ep68AADu0LwAAO7QvAAA7tC8AADu\n0LwAAO6QNhzBhg0bclr/7LPPmhqjoID/UvtUxcZDqVFCJ06cMLU9e/bI9aquxkOdc845cr1K4ala\ncXGxXK+oUVAqrRdLC6qxSyqVuXv3brlevf65c+eamkpQhqA/P5WsjL0nWUaB/S+evAAA7tC8AADu\n0LwAAO7QvAAA7hDYGGPqx+TR7l/zT9Q4ndjYldOnT5ua+tE4Ru1xtnHjxuT1irrXWFgmy4/hOLuo\nz1n9uB/7jqgf9wcHB03twIEDcv3BgwdNTY1imj17tlzf1dWVdKz6jqm/BSGEsG/fPlNT463UHlkh\n6PFY6jX9+uuvcr0KXKjAhnpNMWPxN85cY8yvAABAntG8AADu0LwAAO7QvAAA7hDYGGMNDQ3jcp37\n77/f1ObPny+PPXTokKk9//zzeb+nXMXeu3vvvXec7wT5oqZBqFqWvZ/UhIvY3lmp129tbZXr9+/f\nb2q//fabqakQRK5TN0pKSuSxapqGqh0/flyuX7hwoampaRqVlZVyvQpnjHaPrix48gIAuEPzAgC4\nQ/MCALhD8wIAuEPzAgC4M2W0e6nkqCAXHQ2VbHv11VcLcCfjTyWmQkhPEt11112yvmbNmqT1V155\npawvWbJElXW8DGOuo6Mjp++z+hsU2/NOpQjVuLIdO3bI9S0tLaam9vj66aef5Pq2tjZTU+OZampq\nTC02ckqNYlL7iZ06dUqur6urMzX1Pql7CiGEpUuXmtrixYtNLXb/KkWp/naoEXYh6LRifX39iN9n\nnrwAAO7QvAAA7tC8AADu0LwAAO4Q2BiFN954w9TUOJYs1A/MuY5seuSRR2S9qakpaf2NN94o6/X1\n9aO+pzFEYKNAsgQ2VJBA/Q1SI5tC0EEAFe6I7T119OhRU1MjnzZv3izXf/fdd6am9g5T+341NzfL\nc15wwQWmpl5T7G+1Gu+kQiCxwEZZWZmppb7PsboKZ2TZ44vABgBgQqJ5AQDcoXkBANyheQEA3CGw\ngYmCwEaB5DphQ/2QH/u7pMIBan1s7ywVBFHr1dSMEPReeH/88YepdXZ2mlpFRYU8p5pYowIPsYk3\nasKGWh+bjKPeK/WexCZ8qPvKEs5QCGwAACYkmhcAwB2aFwDAHZoXAMAdmhcAwB0dXwGAcaISgLFk\nXOqxsfFQqftMxUYhqWPVKCY1Biv2mtSxg4ODphZLYKrzqmRgrsnyLPv7jUeKnScvAIA7NC8AgDs0\nLwCAOzQvAIA7BDYAnHVi+3mpugoHxAIbKlzQ19eXdFwI6UGM4eHh5HMqamRTlj0Ds4RYFPU6Ywo0\nYpAnLwCAPzQvAIA7NC8AgDs0LwCAOwQ2AJx1soQAYvtMKSr0oK6V5fpq7yoVmIiFUFSQQgUmYhMu\nUoMYWUIYHvDkBQBwh+YFAHCH5gUAcIfmBQBwh+YFAHCHtCGAs85YjSdSycAs11fXSj3nyZMnk46L\nSb1OCPo+PYx8yoInLwCAOzQvAIA7NC8AgDs0LwCAO1M8/DAHAMD/4skLAOAOzQsA4A7NCwDgDs0L\nAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDg\nDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4M6/\nAXXGDiUZmwcmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7ad7bba90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Training one Autoencoder at a time in a single graph (Tying Weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28*28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # encoding\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.001\n",
    "l2_reg = 1e-5\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "W1 = tf.Variable(initializer([n_inputs, n_hidden1]), dtype=tf.float32, name=\"W1\")\n",
    "b1 = tf.Variable(tf.zeros([n_hidden1,]), name=\"b1\")\n",
    "W2 = tf.Variable(initializer([n_hidden1, n_hidden2]), dtype=tf.float32, name=\"W2\")\n",
    "b2 = tf.Variable(tf.zeros([n_hidden2,]), name=\"b2\")\n",
    "W3 = tf.transpose(W2, name=\"W3\")\n",
    "b3 = tf.Variable(tf.zeros([n_hidden3,]), name=\"b3\")\n",
    "W4 = tf.transpose(W1, name=\"W4\")\n",
    "b4 = tf.Variable(tf.zeros([n_outputs,]), name=\"b4\")\n",
    "\n",
    "h1 = activation(tf.nn.xw_plus_b(X, W1, b1))\n",
    "h2 = activation(tf.nn.xw_plus_b(h1, W2, b2))\n",
    "h3 = activation(tf.nn.xw_plus_b(h2, W3, b3))\n",
    "outputs = tf.nn.xw_plus_b(h3, W4, b4)\n",
    "\n",
    "with tf.name_scope(\"layer1\"):\n",
    "    layer1_outputs = tf.nn.xw_plus_b(h1, W4, b4)\n",
    "    layer1_mse = tf.reduce_mean(tf.square(layer1_outputs - X))\n",
    "    layer1_loss = layer1_mse + regularizer(W1)\n",
    "    layer1_train_op = tf.train.AdamOptimizer(learning_rate).minimize(layer1_loss, var_list=[W1, b1, b4])\n",
    "\n",
    "with tf.name_scope(\"layer2\"):\n",
    "    layer2_mse = tf.reduce_mean(tf.square(h3 - h1))\n",
    "    layer2_loss = layer2_mse + regularizer(W2)\n",
    "    layer2_train_op = tf.train.AdamOptimizer(learning_rate).minimize(layer2_loss, var_list=[W2, b2, b3])\n",
    "    \n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training layer #1\n",
      "0 Train MSE: 0.00377009\n",
      "1 Train MSE: 0.00223285\n",
      "2 Train MSE: 0.00165447\n",
      "3 Train MSE: 0.00153537\n",
      "49% Train MSE: 0.00142765\n",
      "Training layer #2\n",
      "0 Train MSE: 0.0172365\n",
      "19% Train MSE: 0.00936307\n",
      "2 Train MSE: 0.00732472\n",
      "3 Train MSE: 0.00654762\n",
      "49% Train MSE: 0.00575416\n",
      "Test MSE: 0.00601665\n"
     ]
    }
   ],
   "source": [
    "training_ops = [layer1_train_op, layer2_train_op]\n",
    "mses = [layer1_mse, layer2_mse]\n",
    "n_epochs = [5, 5]\n",
    "batch_sizes = [150, 150]\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for layer in range(2):\n",
    "        print(\"Training layer #{}\".format(layer + 1))\n",
    "        if layer == 1:\n",
    "            mnist_hidden1 = h1.eval(feed_dict={X: mnist.train.images})\n",
    "        for epoch in range(n_epochs[layer]):\n",
    "            n_batches = mnist.train.num_examples // batch_sizes[layer]\n",
    "            for iteration in range(n_batches):\n",
    "                print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "                sys.stdout.flush()\n",
    "                if layer == 1:\n",
    "                    indices = np.random.permutation(len(mnist_hidden1))\n",
    "                    hidden1_batch = mnist_hidden1[indices[:batch_sizes[layer]]]\n",
    "                    feed_dict = {h1: hidden1_batch}\n",
    "                    sess.run(training_ops[layer], feed_dict=feed_dict)\n",
    "                else:\n",
    "                    X_batch, _ = mnist.train.next_batch(batch_sizes[layer])\n",
    "                    feed_dict = {X: X_batch}\n",
    "                    sess.run(training_ops[layer], feed_dict=feed_dict)\n",
    "            mse_train = mses[layer].eval(feed_dict=feed_dict)\n",
    "            print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "            saver.save(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    mse_test = mses[layer].eval(feed_dict={X: mnist.test.images})\n",
    "    print(\"Test MSE:\", mse_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_cache_frozen.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa8AAAFsCAYAAAB7FzYbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGnpJREFUeJzt3WtsVNXXx/FdpGXaKRQBhQrhD4qX1isSFUTREGNM8BIl\nxnhFBa+JJpKg0cREfaFvJTFGjEbEW7xENCrxgkEF7xghKmKqgEK0AipCpwyl4PPGJ//ncf223acz\n7XS138/LldnnnJkys3Jyfqxd9ddffwUAADwZVOkLAAAgK5oXAMAdmhcAwB2aFwDAHZoXAMAdmhcA\nwB2aFwDAHZoXAMAdmhcAwB2aFwDAncEVOi8zqVBuVZW+gIGqUCjwfUZZ5fP5Lr/P3HkBANyheQEA\n3KF5AQDcqdQzLwDoE6qq7OOVUreKUuvVedB93HkBANyheQEA3KF5AQDcoXkBANyheQEA3CFtCGBA\nU8nAWNpQJQb379+ftH7QoPR7hQMOOCB5fer1q+uMyZKMLDWZ2V3ceQEA3KF5AQDcoXkBANyheQEA\n3CGwAWBAyxLYSA1n7Nu3L/n8NTU1JR1TBTlU4CMm9T1lWZ8lnNJd3HkBANyheQEA3KF5AQDcoXkB\nANwhsAGg7LJMqChV6tSLmCyBDaW6ujqpFjumem2xWDS1jo4OuV6FMwYPtj/tWT57FbiIXX9vhDPk\neStyVgAASkDzAgC4Q/MCALhD8wIAuEPzAgC4Q9oQQNn1RKowhPSxR1n2vlKvVQnAEHTir7Oz09Ta\n2tpMTY2BCiF9vFI+n09er64plsBMHU8VSxuqv0mpCc4U3HkBANyheQEA3KF5AQDcoXkBANwhsAGg\n16QGJmKBDxU6UPtcqfFKIejARW1tranFwhFq7JIyatQoU4sFFmLX+k+xsIo6bl1dnanFQix79+41\nNRX4KFXsb9rdIAd3XgAAd2heAAB3aF4AAHdoXgAAdwZkYOOTTz4xtYULF8rXjh071tTUA945c+bI\n9SNGjEiqAfivWGBgx44dprZx40ZTU9/xEEL4448/TG38+PGm1tzcLNePHj3a1NT3eejQoaYWe08q\ncJLL5Uxtz549cr0KoahjlrofV5apKVkmbHR3Ggt3XgAAd2heAAB3aF4AAHdoXgAAd2heAAB3qsq9\nx0qiipz0fx155JGm1tLS0iPnamhoMLWpU6f2yLnKbcKECbJ+5513mppKbPWyntlACl0qFApl/z6r\nBF0IIWzdutXU3nrrLVNbsWKFXK/GS6n0cGwM1K5du0xNJQtVgi62n5baO2zIkCFJ5w5Bj6JSxxw3\nbpxcP2vWLFNrbGw0tdhn0hN7d+Xz+S6/z9x5AQDcoXkBANyheQEA3KF5AQDcGZDjoV555RVTW7Nm\njXzt0UcfbWrffPONqX366ady/auvvmpq6gHzxIkTTU2Nvcki9oBVPYzdvHlz8nFVkOOOO+5IXg/8\nX2qfKhVYCEGHI1QAS+1nFUII9fX1pvbTTz+Z2u+//y7Xq8CIqqkQiHpdCPr9t7a2mlpsrJwKbKjf\nODVyKgQ98uq8884ztVgIIxZE+afujoGK4c4LAOAOzQsA4A7NCwDgDs0LAODOgJyw0ZuKxaKpbdq0\nydRUYGPDhg0lnbumpkbWVWBDnX/btm1y/dKlS03tggsuyHh1ZceEjQrJMmFDPbRX/05je0yp79PO\nnTtNTQUzYuffu3dvUi2EENrb201N7TGWz+dNra2tTR5T1VUIIhbYUJ/J7bffbmqFQkGuv//++03t\nlFNOMTUVLAkhfe+w1H3DQmDCBgCgn6J5AQDcoXkBANyheQEA3KF5AQDcGZDjoXqTGsly1FFHJa1t\namoq9+WEEPQoq+3bt5uaShyFEMLZZ59d9mvCwKBSaCqtFhttphJvw4YNM7XYKCS1T5hKO6prCkGP\np1LjmdT5Ozs75TFVClClFWOfyapVq0xNpSKPOeYYub65udnUVDIwSzI9S7Kwu7jzAgC4Q/MCALhD\n8wIAuEPzAgC4Q2CjH4uNg7nwwgtNTY2jefDBB+V6tVcRkCJ1T6dYuEHVVbhCjYwKQQcJVAgktkeV\nWq/CEeqYaoxT7LUqHPHDDz/I9Y8//ripqf3IZs2aJdcPHz7c1NTfKRZiUa9V189+XgCAAY/mBQBw\nh+YFAHCH5gUAcIfARj+2ePFiWW9tbTW1kSNHmtp//vOfcl8SBrjUh/uxcEDq3lExar0KZwwZMiR5\nvZrQoY4ZmzpRXV1tamo/sWXLlsn1n3zyialNnz7d1E444QS5XgVG1PljgYveCGco3HkBANyheQEA\n3KF5AQDcoXkBANwhsNFPqP99P3/+/OT1H3/8samNGTOmpGsC/ik1yJDlgX9sq5BUKnAQm7ChXqvO\nn+X61TE3btxoaitXrpTr6+rqTO2aa64xtcbGRrlehTPU+4+9J/VaFQIpN+68AADu0LwAAO7QvAAA\n7tC8AADu0LwAAO6QNuwnXnvtNVNTKaIQQrj44otN7dBDDy37NQEpsqT9VIotdbxS7LVqj7DYyKnU\nFJ06ZpYE3ldffWVqX375pXztRRddZGrHH3+8qcVGXu3atcvUVAI0yxgu9vMCAECgeQEA3KF5AQDc\noXkBANwhsOGQehi9dOlSU4s9oH3ggQdMrTfGuQBKlsBG6iip2H5g6t95lnCFCnx0dHSYmrr+fD4v\nj/njjz+a2rvvvmtqEyZMkOsvueQSU6utrTW1YrEo16v3lLrHVyVx5wUAcIfmBQBwh+YFAHCH5gUA\ncIfAhkOPP/64qam9fi677DK5nmka6EuyBDZSwxmx9anhjFhgY8+ePaa2e/duU6uvrze19vZ2ecxV\nq1aZ2nfffWdqp59+ulw/adKkpOuMBS7UfmDqc45NyEh9bZYJHSm48wIAuEPzAgC4Q/MCALhD8wIA\nuEPzAgC4Q9qwD1uzZo2s33LLLaY2fPhwU7vvvvvKfk1AKVKThbFkmkoWqteqkUchhDB4sP3JU+eP\nJfNUWjGXy5maSiuuXr1aHvPZZ581tYaGBlM777zz5Ho1Ckpdf01NjVyvxNKaqUgbAgAg0LwAAO7Q\nvAAA7tC8AADuENjoI9SImUsvvVS+Vj20vvzyy02NMVDwQD3cz7KflwphqGBFCDo0kBoCCUGHI9T6\nLVu2mNpDDz0kj7lu3TpTu+mmm0ztmGOOkeuV2CgnRYU7VOAkyzHLHc5QuPMCALhD8wIAuEPzAgC4\nQ/MCALhDYKMC1MPoWbNmmZra0yeEEJqamkzt3nvvLf3CgB6W+tA/Ng1CBQFUiCMWGEid8KFCILH6\njh07TO2FF14wtddff10e84orrjC12bNnm1psaogKXGQJV6QGZmIhGhXuUJ9zlmtKwZ0XAMAdmhcA\nwB2aFwDAHZoXAMAdmhcAwB3ShhXw+++/m9p7772XvP6pp54ytREjRpRySUDFqGSaSrDFXqtkWa9S\ncLFknBo79cMPP5jasmXLTO2UU06Rx5w7d66pjR49Wr5WSR2ZlWWPLjXyKst6dU2xz7S7o6S48wIA\nuEPzAgC4Q/MCALhD8wIAuENgo4f9+eefpjZ16tSktU8//bSsT548uaRrAvoS9cA+y0P8UoMEWcZL\nbdu2zdRefvllU2ttbTW1BQsWyGM2NzebmgpcZBl5pYIl6n3G1qvPNLY+NZxR7j2+uPMCALhD8wIA\nuEPzAgC4Q/MCALhDYKOHPfHEE6a2YcOGpLWnnXaarJd7XxygktQ0DBU4CCH9ob+aEBFC+jSKnTt3\nyvUrVqwwtXfeecfUxo4da2pTpkxJvqYsgYvU/bhin4kSm1DSl3DnBQBwh+YFAHCH5gUAcIfmBQBw\nh+YFAHCHtGGZtLS0yPo999zTuxcCOJMlPatemzryKQSdLNy7d6+pqfFOIYTw9ttvm9pXX31larNn\nzza1uro6eUx1rcVi0dTUtcfWZ9mjLHWPs3KPdyoVd14AAHdoXgAAd2heAAB3aF4AAHcIbJTJypUr\nZT02ZuafmpqaTK22trakawI8yBIESA0i1NTUJK8vFAqmtmXLFrl+06ZNptbY2GhqJ5xwgqnlcjl5\nTBW4UN/92OekRkFl2SPN67g57rwAAO7QvAAA7tC8AADu0LwAAO4Q2KiAU0891dTUnkAENoD/T4UO\nVGAhC7XP1bBhw+RrzznnHFPL5/OmduaZZ5ragQcemHxNWSZkpK6P7eelPr9Sz98buPMCALhD8wIA\nuEPzAgC4Q/MCALhD8wIAuFNVoT1a+tbGMOgP+lYUagApFApuvs+p+1TFEoydnZ1J69XeW9XV1SmX\n+K/nV9R4qSzvqa+lCEMIIZ/Pd3lR3HkBANyheQEA3KF5AQDcoXkBANypVGADAIBu484LAOAOzQsA\n4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAO\nzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0LAOAOzQsA4A7NCwDgDs0L\nAOAOzQsA4A7NCwDgDs0LAOAOzQsA4M7gCp33rwqdF/1XVaUvYKAqFot8nwewv/6yf/6qqtK+jrlc\nrssDcOcFAHCH5gUAcIfmBQBwp1LPvACg16lnMaq2f//+sp9HPRuK6YnnSD2lUtfFnRcAwB2aFwDA\nHZoXAMAdmhcAwB2aFwDAHdKGAAY0lSyMJQNVXdWypBUHDUq7hxg8WP9cq7pKAO7bt0+uT31PWdKS\nvYE7LwCAOzQvAIA7NC8AgDs0LwCAOwQ2yuSZZ56R9UKhYGpffPGFqT366KPJ57r77rtNbebMmaZ2\n5plnJh8T6E9i4QIVpFDf0R07dsj1P/30k6l9++23pvbLL78knTuEECZOnGhqRx99tKk1NzfL9fX1\n9aamAhudnZ1yvVLqeKveGBnFnRcAwB2aFwDAHZoXAMAdmhcAwJ2qCv2v6b71X7Uzuvnmm01t0aJF\nFbiS/1IPc1etWiVf29DQ0NOXUwl9c7OjAaBYLFb0+6x+wzo6OuRrt27damqrV682tU8//VSu//zz\nz01t8+bNptbe3m5q48aNk8esrq42talTp5raTTfdJNdPmjRJ1v8p9pkosWkeSmoPyRLiyOVyXb6Y\nOy8AgDs0LwCAOzQvAIA7NC8AgDs0LwCAO4yH6kJPJAsnT55sarNnzza1lpYWuf7JJ580tXXr1pna\nSy+9JNfPnTu3q0sEShJLoKUmzmKvU3U1dqmtrU2u//DDD01tyZIlprZx40a5fsKECaZ23XXXmVpj\nY6Optba2ymO+8cYbpvbdd9+ZmhpDFUII48ePN7W6ujpTi30mQ4YMMTWVNoylFSs1Soo7LwCAOzQv\nAIA7NC8AgDs0LwCAOwQ2/qb26QkhhMceeyxp/UknnSTrb775pqmph6k1NTWmtm/fPnnM77//3tTU\ng+jt27fL9UBPiz2EVw/yszzwV69V35Nff/1Vrl+xYoWprVmzxtSuuuoquf7WW281tUMOOcTU1PWr\nPb5C0NeqRlbF9hhL/UxVMCMEvc9Xlr9Jajhj0KDy3itx5wUAcIfmBQBwh+YFAHCH5gUAcIfAxt9i\n4Qb1MFKFM5YvXy7X19fXd/uaFi9eLOtqTyHlggsu6Pa5gZ6gggDqQX4s8KFeqyY/FItFuX7kyJGm\nNm/ePFObM2eOXD969GhTU9Modu/ebWpbtmyRx3z//fdNTe0HNmXKFLk+n8+bmpo6okJhIejAi3pP\nscCF+vy7OzUjC+68AADu0LwAAO7QvAAA7tC8AADu0LwAAO6QNvzbiSeeKOsqhahSO7W1tWW/ptho\nqti+OkB/odJyIYRwwAEHJK1XqcAQQrjxxhtNbcyYMaYWS+apukrW7d2719TWr18vj/nzzz+b2rHH\nHmtqKikZgv49UGnHWFowl8uZmkobxsZAqb9Jlv28uos7LwCAOzQvAIA7NC8AgDs0LwCAOwQ2utDQ\n0NAr53nqqadMbe3atcnrzz77bFM77LDDSromoNzUg3wVzsiyn5fap0rtsRWCDheo8+/Zsyd5vdpn\n66OPPjK1Dz74QB5TjXeaMWOGfK2irr+6ujqpFoL+rNXIqNj+gql/v1hgpLujpLjzAgC4Q/MCALhD\n8wIAuEPzAgC4Q2CjAr788ktTu+GGG0wt9tC4sbHR1BYuXGhqsQe0QE+LBS5UPcsDe/VaFSSIhQPU\n5InOzk5TiwW11Pm3bdtmaitXrjQ19b0PIYQrr7zS1M4991xTGzZsmFyfOvVDvc8Q0kM0ampI7Fyp\ntVJw5wUAcIfmBQBwh+YFAHCH5gUAcIfmBQBwh7RhBXz88cemFksWKmpPoiOOOKKkawLKKUuyTKXd\nYvt2qcRblmSdovbii53/jz/+MLXly5eb2ksvvWRqTU1N8phnnHGGqan9yOrr6+V6lbZUvyexPdLU\n+4+lNVOpv0mWkV8puPMCALhD8wIAuEPzAgC4Q/MCALhDYKOHXXvttab2/PPPJ6297bbbZP32228v\n6ZqAviTLKCEVpFDhgmKxKNfncjlTU/uB/fbbb3L9smXLTG3JkiWmVigUTG3mzJnymJMmTTK1UaNG\nJR0zBB3EUCGMWIhF1VM/51g9Fs4oJ+68AADu0LwAAO7QvAAA7tC8AADuVPXGgzWhIiftSW1tbbJ+\n+OGHm9rWrVtNTf2P+q+//loec8SIERmvbkAo72ZBSFYsFkv6PmcJbKhwwuDBNnfW3t4u16sgg/oN\n/Oijj+T65557ztTWrl1rameddZapXX/99fKY6ruvrikWuFD79qnPL7a/X5bpPkrqNI0sEzZyuVyX\n32fuvAAA7tC8AADu0LwAAO7QvAAA7jBho0wuvvhiWVfhDOXWW281NYIZGKhigQ0VWlCvrampST6u\nCne89957cv2bb75pauPGjTM1NTVDBUtCSA83xNarz0SFMIYOHSrXqy1V1NYzajpJCHoaR5YtabqL\nOy8AgDs0LwCAOzQvAIA7NC8AgDs0LwCAO6QNu+GLL74wtVg6SbnoootMbf78+aVcEtCvqDFQsbpK\nxsXShjt37jS1b775xtTWrVsn16u9v6ZNm2Zq06dPNzU1BioEff0qrRdL+5W6H5dKW6q0o3rv/3bc\nf4olSLuLOy8AgDs0LwCAOzQvAIA7NC8AgDsENrqwe/duU7vzzjtNraOjI/mYU6ZMMbXYA2ZgIIrt\n/aTqKpwQW//bb7+Z2vLly01t1apVcv3JJ59saueff76pHXzwwXK9UiwWTU3tvRXbd0sFJtQoKRVW\nib1W/R6pzzmE9JFd5d47kjsvAIA7NC8AgDs0LwCAOzQvAIA7BDa68Mgjj5jau+++m7z+2muvNTWm\naQD/LjaNQQUZFBW0CiGE9evXm9rnn39uarFpEmqaRlNTk6mpcEOhUJDHVO9J1dQkjti51CSS2H5g\ntbW1SceMfabqXKl/p1Jw5wUAcIfmBQBwh+YFAHCH5gUAcIfmBQBwp6rcIzsSVeSk3aGSOFlGQf35\n55+mVl9fX9I1QSrvZkFIViwWk7/P6vdGjTeK/S6pZNu+fftMbevWrXL9okWLTO311183tZkzZ8r1\nKik8duxYU0t9n7F6bBSTos6lPpNYglOlENUYrc2bN8v1+Xze1MaMGZN0nth15XK5Lr/P3HkBANyh\neQEA3KF5AQDcoXkBANxhPFQPa2trM7XYg9tSqHE2sYe+6mFubK8gRY2JWbhwYfJ6RV3rXXfdJV/b\nG6NnUJosQbAsez+pwIb6txMbpbRx40ZT27RpU9I1haD3xFLnUvthxY65a9cuU1O/EePGjZPr1X5a\nKrCivvchhPDLL7+Y2meffWZqsfFQZ5xxhqkdeOCBphb7PYp9Ll3hzgsA4A7NCwDgDs0LAOAOzQsA\n4A6BjR6m/vd9T7jxxhtN7ZBDDpGvbW1tNbWHH3647NdUqthnN2/evF6+EmSV5SF8qeEOFVYaNWqU\nXK/+TakgwurVq+V6FfhQ1N5d48ePl68tFoumdtxxx5nahAkT5Hq1R5kKocRs2LDB1Nrb201t8uTJ\ncn3qNJDuBjNiuPMCALhD8wIAuEPzAgC4Q/MCALhD8wIAuMN+Xl1QybYnnniiAlfS+2L776Smi66+\n+mpZnzZtWtL66dOny/qhhx6qyuznVSGl7uelxJJpaoyZShvGxkO9+OKLpvbaa6+ZmtqHLwQ9ymn7\n9u2mpsZDxUawqbSj2vMv9pmoUVKNjY2mFksfq7/JyJEjTe3YY4+V62fMmGFqKtWp/k4x7OcFAOiX\naF4AAHdoXgAAd2heAAB3CGx0w5IlS0yto6OjpGOuXbvW1Eod2bRgwQJZnzRpUtL6888/X9YPPvjg\nbl9TDyKwUSGlBjZULbbnndq7Su1TpQITIejRaO+//76pxfauUmOn1DV9++23ptbS0iKPqfb8U/th\nHXbYYXL90KFDTe2ggw4ytSOPPFKuVwEsVWtoaJDrhw8fbmrq808NeoVAYAMA0E/RvAAA7tC8AADu\n0LwAAO4Q2EB/QWCjQrIENpQs+zypcIb6DYtNc1CvVdM4VAgjhBDq6upMTYVL1H5YsWPu37/f1Kqr\nq01NTd0IQU/9UOGIWIhFnV9da6xXqPev/qZZ/s4ENgAA/RLNCwDgDs0LAOAOzQsA4A7NCwDgjt6w\nCQAqKJZMUyk69dpisSjXpyb7YlQyUSUg1XgpNcYpBL1vnho3p649BJ2sVAnA2Him1M9Uvc+YLMnC\n7uLOCwDgDs0LAOAOzQsA4A7NCwDgDoENABWVusdXTCzIoKSOl4pRgQ0VeFBjpGIhhkKhkHSe2PtU\n51fvKba+1HBFb4QzFO68AADu0LwAAO7QvAAA7tC8AADuENgA4EZqOCD2OjVNIzZ5QlGTK1LDEbEJ\nFer86jyqFju/ev9Zgi2l7sfVG7jzAgC4Q/MCALhD8wIAuEPzAgC4Q/MCALhD2hCAG6mjnGLJPCVL\niq6U88fWqv28UhOEsXqWkVd9LUWYijsvAIA7NC8AgDs0LwCAOzQvAIA7VVke7AEA0Bdw5wUAcIfm\nBQBwh+YFAHCH5gUAcIfmBQBwh+YFAHCH5gUAcIfmBQBwh+YFAHCH5gUAcIfmBQBwh+YFAHCH5gUA\ncIfmBQBwh+YFAHCH5gUAcIfmBQBwh+YFAHCH5gUAcIfmBQBwh+YFAHCH5gUAcIfmBQBwh+YFAHCH\n5gUAcIfmBQBw538AH/ARpijPUtIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7af067b38>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_cache_frozen.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Unsupervised pretraining for mnist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150\n",
    "n_outputs = 10\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 5e-4\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "y = tf.placeholder(tf.int32, [None])\n",
    "\n",
    "W1 = tf.Variable(initializer([n_inputs, n_hidden1]), dtype=tf.float32, name=\"W1\")\n",
    "b1 = tf.Variable(tf.zeros([n_hidden1,]), name=\"b1\")\n",
    "W2 = tf.Variable(initializer([n_hidden1, n_hidden2]), dtype=tf.float32, name=\"W2\")\n",
    "b2 = tf.Variable(tf.zeros([n_hidden2,]), name=\"b2\")\n",
    "W3 = tf.Variable(initializer([n_hidden2, n_outputs]), dtype=tf.float32, name=\"W3\")\n",
    "b3 = tf.Variable(tf.zeros([n_outputs,]), name=\"b3\")\n",
    "\n",
    "h1 = activation(tf.nn.xw_plus_b(X, W1, b1))\n",
    "h2 = activation(tf.nn.xw_plus_b(h1, W2, b2))\n",
    "logits = tf.nn.xw_plus_b(h2, W3, b3)\n",
    "\n",
    "cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y)\n",
    "reg_loss = regularizer(W1) + regularizer(W2) + regularizer(W3)\n",
    "loss = reg_loss + tf.reduce_mean(cross_entropy)\n",
    "\n",
    "train_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
    "\n",
    "correct = tf.nn.in_top_k(logits, y, 1)\n",
    "accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "pretrain_saver = tf.train.Saver([W1, W2, b1, b2])\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train accuracy: 0.953333 Test accuracy: 0.9358\n",
      "1 Train accuracy: 0.993333 Test accuracy: 0.9469\n",
      "29% Train accuracy: 0.973333 Test accuracy: 0.9476\n",
      "3 Train accuracy: 0.973333 Test accuracy: 0.9323\n"
     ]
    }
   ],
   "source": [
    "# regular training\n",
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = np.random.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(train_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\" \")\n",
    "        saver.save(sess, \"./my_model_supervised.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"Test accuracy:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_cache_frozen.ckpt\n",
      "0 Train accuracy: 0.926667\tTest accuracy: 0.9125\n",
      "19% Train accuracy: 0.966667\tTest accuracy: 0.939\n",
      "2 Train accuracy: 0.98\tTest accuracy: 0.9249\n",
      "3 Train accuracy: 0.966667\tTest accuracy: 0.9286\n"
     ]
    }
   ],
   "source": [
    "## with pretraining\n",
    "n_epochs = 4\n",
    "batch_size = 150\n",
    "n_labeled_instances = 20000\n",
    "\n",
    "#training_op = optimizer.minimize(loss, var_list=[weights3, biases3])  # Freeze layers 1 and 2 (optional)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    pretrain_saver.restore(sess, \"./my_model_cache_frozen.ckpt\")\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = n_labeled_instances // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            indices = np.random.permutation(n_labeled_instances)[:batch_size]\n",
    "            X_batch, y_batch = mnist.train.images[indices], mnist.train.labels[indices]\n",
    "            sess.run(train_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train accuracy:\", accuracy_val, end=\"\\t\")\n",
    "        saver.save(sess, \"./my_model_supervised_pretrained.ckpt\")\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})\n",
    "        print(\"Test accuracy:\", accuracy_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacked denoising autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 300\n",
    "n_hidden2 = 150  # codings\n",
    "n_hidden3 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "l2_reg = 1e-5\n",
    "dropout_rate = 0.3\n",
    "\n",
    "activation = tf.nn.elu\n",
    "regularizer = tf.contrib.layers.l2_regularizer(l2_reg)\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "is_training = tf.placeholder_with_default(False, shape=[], name=\"is_training\")\n",
    "X_drop = tf.layers.dropout(X, dropout_rate, training=is_training)\n",
    "\n",
    "W1 = tf.Variable(initializer([n_inputs, n_hidden1]), dtype=tf.float32, name=\"W1\")\n",
    "b1 = tf.Variable(tf.zeros([n_hidden1,]), name=\"b1\")\n",
    "W2 = tf.Variable(initializer([n_hidden1, n_hidden2]), dtype=tf.float32, name=\"W2\")\n",
    "b2 = tf.Variable(tf.zeros([n_hidden2,]), name=\"b2\")\n",
    "W3 = tf.transpose(W2, name=\"W3\")\n",
    "b3 = tf.Variable(tf.zeros([n_hidden3,]), name=\"b3\")\n",
    "W4 = tf.transpose(W1, name=\"W4\")\n",
    "b4 = tf.Variable(tf.zeros([n_outputs,]), name=\"b4\")\n",
    "\n",
    "h1 = activation(tf.nn.xw_plus_b(X_drop, W1, b1))\n",
    "h2 = activation(tf.nn.xw_plus_b(h1, W2, b2))\n",
    "h3 = activation(tf.nn.xw_plus_b(h2, W3, b3))\n",
    "outputs = tf.nn.xw_plus_b(h3, W4, b4)\n",
    "\n",
    "mse_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "reg_loss = regularizer(W1) + regularizer(W2)\n",
    "loss = mse_loss + reg_loss\n",
    "\n",
    "train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.0188708\n",
      "1 Train MSE: 0.0135786\n",
      "29% Train MSE: 0.0113397\n",
      "3 Train MSE: 0.010279\n",
      "4 Train MSE: 0.00961263\n",
      "5 Train MSE: 0.00885574\n",
      "69% Train MSE: 0.00939195\n",
      "7 Train MSE: 0.00923494\n",
      "8 Train MSE: 0.00853442\n",
      "9 Train MSE: 0.0090277\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, _ = mnist.train.next_batch(batch_size)\n",
    "            sess.run(train_op, feed_dict={X: X_batch, is_training: True})\n",
    "        mse_train = mse_loss.eval(feed_dict={X: X_batch, is_training: False})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_train)\n",
    "        saver.save(sess, \"./my_model_stacked_denoising.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_stacked_denoising.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa8AAAFsCAYAAAB7FzYbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGyhJREFUeJzt3VuMVvXVx/E/yswwB5hhkJNjoSqiA8YKVcGi1BhLTDTi\nISYeLlQk0ZjYpCZto0mT1hsvlRujRqP1cGFiYlsvFE+1gBEPIERRERGV82lkZM4H7I3mfV/Wb73z\nfw7DM2vm+7lcef5772fjM6s7+9f1H/fjjz8mAAAiOanSFwAAQKFoXgCAcGheAIBwaF4AgHBoXgCA\ncGheAIBwaF4AgHBoXgCAcGheAIBwaF4AgHDGV+i8zKRCuY2r9AWMVW1tbfyeUVbNzc1D/p558gIA\nhEPzAgCEQ/MCAIRTqXdeADBijRunX7mUsoXUcByz0HNV8vzlPg9PXgCAcGheAIBwaF4AgHBoXgCA\ncGheAIBwSBsCGDNyE2/e5046yf7vffVZ9bljx47JY/b19Zna4OCgqVVXV8v148fbP+Mnn3xy1nV6\ndVXzEozqew1HgvF4PHkBAMKheQEAwqF5AQDCoXkBAMIhsAFg1FGBCY8KIqgQhMcLYuScJyV9raqm\nQhgp6WutqqrKPn9/f7+pqcCIqlUST14AgHBoXgCAcGheAIBwaF4AgHAIbAAYcQoJXJT62UKmUfT2\n9ppaT0+PqampGSpE4V2T4l2TOn9XV1f2+VXgJHdqh7dehUNygy25ePICAIRD8wIAhEPzAgCEQ/MC\nAIRD8wIAhEPaEEBFeWOLFJWCG45kovc5lbhT11RTU2Nq3nglNZ5JJQMLGVmljqkSkCnp79rR0WFq\nEyZMkOtLHa9VLJ68AADh0LwAAOHQvAAA4dC8AADhENgAUBL1wt4bZaSoIIM3ikh9Vp2/s7NTrldB\nhOrq6uzzq8/W1dVlrffuSanjmVRdHdMLjAwMDJiad/9yz6+O6X3/QgI3/2ddUasAAKggmhcAIBya\nFwAgHJoXACCcMRnYWL9+vamtWrVKfralpcXUamtrTe22226T65ubm7NqQFSFhDNygwTd3d1yvfrs\nvn37TO2jjz6S6/fv329q06ZNMzUVwkgppaamJlM75ZRTTE2FGLz9tFRdhVBUWMQ7l6p530kdVwUu\nvEkouVNP2M8LADDm0bwAAOHQvAAA4dC8AADh0LwAAOGMKyQpVEYVOenPzj77bFPbtm3bsJyrsbHR\n1BYvXjws5yq3X/7yl7J+//33m9qsWbOG+WqGlL8pFMqqra3N/J69ZJpKoam/Qb29vXL9gQMHTO2T\nTz4xtQ8++ECuV8lEtU9Ve3u7XK+uS30nlcDr6emRx1QJRnX+qVOnyvXq+lVa8LTTTpPr1d+jOXPm\nmJpKVaakv6tKFhaSNmxubh7y98yTFwAgHJoXACAcmhcAIByaFwAgnDE5Huof//iHqW3atEl+dv78\n+aa2ZcsWU3v//ffl+n/+85+mtnr1alM7/fTTTW3Hjh3ymLnUi9SUUpo5c6ap7dy5M/u4Ksjx5z//\nOXs9RpdSQ19qlFFDQ4P8rAqC7N6929RmzJgh16vQgwpSqBFwKaW0a9cuU1PjlXL3HUtJh1Dq6+tN\nzbvP6lzqb4cKq6SkgxxnnXWWqXnjrVRgRQVGyo0nLwBAODQvAEA4NC8AQDg0LwBAOGNywsaJpF4G\nf/PNN6amAhtff/11Sef29v9RgQ11/oMHD8r1L7/8sqktX768wKsrOyZsVMjhw4ezJ2yocEYhe1cp\nbW1tptbR0ZF9/v7+flPbu3dv9rnU5AkVYjhy5Ig8ppo8UVNTY2qTJk2S6zdu3Ghqb7zxhql5Aa47\n7rjD1M4//3xTU5NAPCpE4k3YUD2ICRsAgFGJ5gUACIfmBQAIh+YFAAiH5gUACGdMjoc6kdReO+ec\nc07W2tbW1nJfTkpJj7I6dOiQqS1atEiuX7ZsWdmvCXF5yUIlN92s0mop6RFFas+8urq6kq5J/W5T\n0nsBqgSjuk7vu6u0o7p+L0G5bt06U1PJPm8/r1/84hemptKO3vWrc3n/fuXEkxcAIByaFwAgHJoX\nACAcmhcAIBwCG6NYZ2enrF933XWmpl66PvLII3K9t9cRMBQVTlCjoLwQiPrvVNUKGXun1nuBDTVi\nSY2CUrWuri55TG+frON54+K+/PJLU+vt7TU1LwA2ffr0oq8pJR3OUDUVbCkFT14AgHBoXgCAcGhe\nAIBwaF4AgHAIbIxizzzzjKzv27fP1KZMmWJqs2fPLvclYYzz9pQ63sDAgKyrIIQKd6jPpaQDI4WE\nO1QQIjeI4F2TCofs37/f1P71r3/J9Rs2bDC1JUuWmNrvfvc7uV7tE6bufyGTVAr5bLF48gIAhEPz\nAgCEQ/MCAIRD8wIAhENgY5TYvn27qd13333Z69977z1TmzFjRknXBORQEy68cIP6rApMeCEKtb4Q\nahqICqGoCROFBEM+++wzU1PBjJR0OOKSSy4xtalTp8r13r0+ngq7pFT+yRm5ePICAIRD8wIAhEPz\nAgCEQ/MCAIRD8wIAhEPacJR45ZVXTM1LB914442mdsYZZ5T9moAcKi2n0npevaamxtRUKtDT09Nj\nal4qsa6uztS8UVbHU9eZUkrff/+9qW3evNnU9uzZI9dfeumlprZ48WJT8+6J+v6q5q3PTVF6I6MK\nSWH+bzx5AQDCoXkBAMKheQEAwqF5AQDCIbARkApivPzyy6bmvSB+6KGHTK1SI14wtuS+nO/r68s+\npvpsqXtPeevVb0+FO9QeXd4x1Wg3NQpKHTOllK6++mpTa2lpybrOlFLq6uoyNTUyyhsjlbvHmnf+\nYvf+4skLABAOzQsAEA7NCwAQDs0LABAOgY2AnnrqKVNbu3atqd1yyy1yPdM0MJKoEIcXIKqqqjI1\nFQRQEyI86pheOEJdq5qwoUIMBw8elMdcs2aNqX366aem9tvf/lauX7Rokamp76SCGd5nS1VICIYJ\nGwCAMYPmBQAIh+YFAAiH5gUACIfmBQAIh7ThCLZp0yZZv/fee02tqanJ1B588MGyXxNQitxRQF7a\nUK1Xn/X2shs/3v7JU5/10ooqGVdfX29qKtn3+uuvy2O+8847pqbGOy1fvlyuV+dX38nbd0yNkVNp\nSS8VqOpqZJc3XqpYPHkBAMKheQEAwqF5AQDCoXkBAMIhsDFCdHd3m9rNN98sPzs4OGhqt956q6kx\nBgojjQpcqHCBF+xQY5vUeChvLzt1LvV78vaeUqOUent7Te2TTz4xNbXnXkoptbW1mdo111xjaued\nd55cr6716NGj8rOKCryocIW6Tyn5QZDc9cXuJciTFwAgHJoXACAcmhcAIByaFwAgHAIbFaBesF51\n1VWmtnXrVrm+tbXV1P72t7+VfmHAMFPTGArZ+0lRQQBvwob67alwgreflwoX7Ny509SeffZZU9u4\ncaM85tKlS03tyiuvNDU1RSclHZhQ90RNF/HqhdxTdf/Uv1+x+3Z5ePICAIRD8wIAhEPzAgCEQ/MC\nAIRD8wIAhEPasALUOBi1p4/nueeeM7Xm5uZSLgk4IXLTht54ptx9orxknErRVVdXZ11TSil1dnaa\n2vr1603t3//+t6nV1dXJY6pk4bx580zN2w9L3avcMVop5Y/HUqOxvPXlThYqPHkBAMKheQEAwqF5\nAQDCoXkBAMIhsDHM2tvbTW3x4sVZa59//nlZX7BgQUnXBIwkKhzhhQOUQsIF6lxqPJIXONi9e7ep\nrV27Nuv8N9xwgzzmsmXLTE2FSLwQivqsGmPlrc8dL+WtV981N5hTCp68AADh0LwAAOHQvAAA4dC8\nAADhENgYZk8//bSpff3111lrL7nkElkv94tPYKTxAhMqnKHCCaqWkp480dvba2oqaJVSSm+//bap\nvfvuu6ZWW1traueff748ZkNDQ9Y1qRBFSvmBCW/ChlLI1JPc9eXGkxcAIByaFwAgHJoXACAcmhcA\nIByaFwAgHNKGZbJt2zZZ/+tf/3piLwQYxbzE3fHUyCSvrpJxu3btkuu3bNliauq3f9lll5laS0uL\nPKZKBqp9w1RSMiU93io3lVkIbz+xE7F3l8KTFwAgHJoXACAcmhcAIByaFwAgHAIbZaL29EkppR9+\n+CFrfWtrq6mpETPAWOCNIlKhARVE8EIEfX19ptbR0WFq+/fvl+t37txpajNmzDA1Fc5oamqSx1Tf\naeLEiaamghleXY2X8u5pqXtv5QY2vGMWG/jgyQsAEA7NCwAQDs0LABAOzQsAEA6BjQr4zW9+Y2pv\nvPGGqRHYQFTlfjn/MxVuUEGEgYGB7OtS1+T99q644gpTW7p0qampvbu8Y6oQiQqheNNF1PdXtULu\n/XBMzSj3MXnyAgCEQ/MCAIRD8wIAhEPzAgCEQ/MCAIQzrkJ7sVRmAxiMZvnzbFBWbW1tFf095yYI\nC/msl1b0Riwdr6qqytQK2U8r9zwp5Y9yqtS+W8Vobm4e8kvx5AUACIfmBQAIh+YFAAiH5gUACKdS\ngQ0AAIrGkxcAIByaFwAgHJoXACAcmhcAIByaFwAgHJoXACAcmhcAIByaFwAgHJoXACAcmhcAIBya\nFwAgHJoXACAcmhcAIByaFwAgHJoXACAcmhcAIByaFwAgHJoXACAcmhcAIByaFwAgHJoXACAcmhcA\nIByaFwAgHJoXACAcmhcAIByaFwAgHJoXACAcmhcAIByaFwAgnPEVOu+PFTovRq9xlb6Asaq9vZ3f\nc4Zx4/R/oj/+yO07XmNj45C/Z568AADh0LwAAOHQvAAA4VTqnRcAlIX3LklR75fUeu+Yx44dK2l9\n7jWddJJ9rij1mIW8Wyv1PZy61nK/2+PJCwAQDs0LABAOzQsAEA7NCwAQDs0LABAOaUMAo46XzFP1\nwcFBU+vt7ZXrVQpQpehOPvnkrJq3vqqqytTGj9d/rtX6gYEBU1Pf01tfagLzREwN4ckLABAOzQsA\nEA7NCwAQDs0LABAOgY0yeeGFF2S9s7PT1DZs2GBqTzzxRPa5/vKXv5ja5ZdfbmqXXXZZ9jGBSil1\nlJBar0IEKaXU399vavv37ze1PXv2yPXbt283tb6+vqyaF7hoaGgwtdbW1qxaSinV1taamgp8eNQ9\nKWQUVaXw5AUACIfmBQAIh+YFAAiH5gUACGfcifh/QgsVOWm53HPPPab2+OOPV+BK/se8efNMbd26\ndfKzjY2Nw305lTDy3zCPUu3t7SX9ngsJB6gJF0pPT4+st7W1mdrq1atNbc2aNXK9CnKo35MKQXR3\nd8tjqmkY8+fPN7UrrrhCrr/00ktNbfLkyabm/a1XoTIVAqmurpbr1Xf1AjO5Ghsbh/yPgicvAEA4\nNC8AQDg0LwBAODQvAEA4NC8AQDiMhxrCcCQLFyxYYGo33HCDqW3btk2u//vf/25qn332mam99NJL\ncv2dd9451CUCFeWlClXiTe29dejQIbn+rbfeMrUXX3zR1Hbs2CHXz5kzx9SamppMbdq0aabW3t4u\nj/npp5+amhpZtXfvXrm+vr7e1NTeYV4CU6Uga2pqTE2lIlPK3/ur3Ml2nrwAAOHQvAAA4dC8AADh\n0LwAAOEQ2PjJd999J+tPPvlk1voLL7xQ1l977TVTq6urMzX1InpwcFAe86uvvjK1d99919S8l9bA\nSKJe5HuBDRXOUEEE77/9zZs3m5r6Pa5cuVKuP++880xt1qxZpjZhwgRT++KLL+Qx1XgmNYbqyJEj\ncr0Kd8ycOdPUvDFcaj8xtR9ZIXuEqcCIF/goFk9eAIBwaF4AgHBoXgCAcGheAIBwCGz8xHvBq14m\nq3DGm2++Kderl6G5nnnmGVn/8MMPs9YvX7686HMDwyF38oI3jWH8ePsnS4WdvP2kZs+ebWpLliwx\ntUWLFsn1au8u9RtXIZKtW7fKYx48eNDUVFhr6tSpcr06l/r+XmBDTehQ00BUWCYlHU5RvBBOsXt/\n8eQFAAiH5gUACIfmBQAIh+YFAAiH5gUACIe04U8WLlwo6yqFqNJNtbW1Zb8mbzSVGt0CRKBShCpB\nqGqFrJ8+fbpcf/HFF5vamWeeaWqTJk2S69VvT9XUKCc1Ki4lPTZq6dKlWdeZUkpTpkwxNTWKSf3d\n8j6rxjt5aUGVjFTJRtKGAIAxj+YFAAiH5gUACIfmBQAIh8DGENQ4mOHw3HPPmZrae8izbNkyU/Ne\n8ALDzRtFpF7Oqxf+KjCQUko1NTWmpsYWtbS0yPUTJ040NRV4UCOXvLqqffTRR1k1j7omVUsppcmT\nJ5uauqdeYEL9W6mRUd6/iaLOX2www8OTFwAgHJoXACAcmhcAIByaFwAgHAIbFfDxxx+b2l133WVq\n3v45M2fONLVVq1aZWlVVVRFXB5TO249LhQPUi3zv5X6p0yByA1jqPN51bdmyxdTWrVtnagcOHJDH\n/NWvfmVqauKP2ossJf07V/e5v79frlchGHVPC/k39c5VTjx5AQDCoXkBAMKheQEAwqF5AQDCoXkB\nAMIhbVgB7733nql5yULl7rvvNrW5c+eWdE1AOZU6HspLtqnPdnd3Z33Ok5vWSymlw4cPm9p//vMf\nU3v11VdNTY1xSiml+fPnm9qcOXNMra6uTq5Xyb7cfce89SqB6FH/Vqrm3dNi8eQFAAiH5gUACIfm\nBQAIh+YFAAiHwMYwW7Fiham9+OKLWWv/8Ic/yPqf/vSnkq4JGG6FjBJSNS9woepqlJO3H5cKPYwf\nb/8MHjlyRK5Xe+ytX7/e1Pbu3WtqK1eulMe84IILTO2cc84xNW8/LfX9VTjDW6/uaXt7u6nV1tbK\n9dXV1aZWyN5dxQY5ePICAIRD8wIAhEPzAgCEQ/MCAIRDYKNMOjo6ZF39P+3Vy+Tp06eb2gMPPCCP\nqV6QAlGpvbcKCXyoCRldXV3Z51K++uorWX/77bdN7fPPPze11tZWUzv77LPlMRcsWGBqKhzhhVBU\nOELtW+aFKFRd3X9vjzNVz73PpeDJCwAQDs0LABAOzQsAEA7NCwAQDoGNMrnxxhtl/cCBA1nrf//7\n35tac3NzSdcEjDS5W2V4gY3c9ZMmTZLrVZDg6NGjpvbmm2/K9W+99ZapTZkyxdSWLl1qameddZY8\n5sSJE01NTfjwtjRR25eoUJc3YUPdUzV1xAt8qAkd6rPqmN75c/DkBQAIh+YFAAiH5gUACIfmBQAI\nh+YFAAiHtGERNmzYYGrvvPNO9vrrr7/e1O67775SLgkIITdZ2N/fL9dPmDDB1FSKTX0upZTa2tpM\nTe3H9fHHH8v1ahTSsmXLTO2mm24ytdmzZ8tjqrSiSil790Td0+7ublPr7e2V6xsaGkxNjdzyxlOp\nFGMhCdJi8eQFAAiH5gUACIfmBQAIh+YFAAiHwMYQ1IvP+++/39S80S3Kr3/9a1Njjy6MJuqFfUr6\npX0h4QC1XgUGvHDDjh07TE2Frb788ku5fuHChaZ27bXXmpoaBVVXVyePqf52qMCJd0/V/VP3xNtj\nyxvbdDy1x1hKKXV2dmZdUyF7tOXgyQsAEA7NCwAQDs0LABAOzQsAEA6BjSE89thjpqb29PGsWLHC\n1JimgdGu1GkKhYQL1Ge9ANW3335rau+//76pqcBBSilddNFFpqYmZ6jASEdHhzxmbjijsbFRrlf7\neanze4ERdU/Vv5+aLuLV1f0rNpjh4ckLABAOzQsAEA7NCwAQDs0LABAOzQsAEA5pwyE88MADJa1/\n+OGHTY1RUBircvd58pJpauyRSssdOXJErt+0aZOpqRFwp59+uly/YMGCrPOr7+QlMAcHB01NJQi9\nvxsq2afShkePHs1e397ebmq7d+/OXq/uXyEjw3Lw5AUACIfmBQAIh+YFAAiH5gUACIfAxjBTI2G8\n0TelUC941cvtlPQL4t7e3uxzqRfcq1atyl6vqGv1wjLe6B6MfMeOHTO13JFPKemX++q/3X379sn1\nBw4cMDX1e2hubpbrVRBCBRnUKCbvv1s1Hqqrq8vUvMCDOteePXtMbfv27XK9Crfs2rXL1E455RS5\n/sILLzS1mTNnmpr6nqXgyQsAEA7NCwAQDs0LABAOzQsAEA6BjWHW0tJyQs5z9913m9qpp54qP6te\nZj/66KNlv6ZSefdu5cqVJ/hKUC4qdKBCHN7LfRXsUftJHT58WK5X/+2raRIqsJBSSqtXrza1np4e\nU1P7idXW1spjquuvr683NW/CRltbm6mp76SCVt56FQBT00VSSuncc881teEIpZlzDPsZAAAoM5oX\nACAcmhcAIByaFwAgHJoXACAc0oZDuPXWW03t6aefrsCV/P8ee+yxsh9Tje1JyR87dbzbb79d1i++\n+OKs9UuWLMn6HOJQ450KSeapUU7qmLNmzZLrFy5caGqHDh0yNTVeKSWdQlSjnFQCUSUAU9IpQjWe\nSo2mSimlhoaGrPXTp0+X66dNm2ZqjY2NpqbuXUopzZkzx9TUyKpy48kLABAOzQsAEA7NCwAQDs0L\nABDOOPWy8wSoyEnL5dlnnzU19dK5EJs3bza1Ukc2/fGPf5R19YJVueaaa2RdveAdAfRmRxh27e3t\nJf2eVRDBCwt5e1odzxulpAIXH374oamp32NKeu8wFfhQVNgkJf2d1PdXY7RS0ntnzZ0719Tmz58v\n16t7pf5NJk2aJNc3NTWZmgpsFDIyqrGxcch/aJ68AADh0LwAAOHQvAAA4dC8AADhENjAaEFgo0JK\nDWyov0HeNAn1WRVkUFMnUkqpqqrK1NQ+Vz/88INcr8INauJMZ2dn1nV6x1SfVWGRlPR3nThxoqmp\n7+4dV4VLvPOrvddygzUeAhsAgFGJ5gUACIfmBQAIh+YFAAiH5gUACIf9vACMON54KJUMVLy0oErG\nqbFFXlpRXZdKQNbX15ual/ZTVFrR++7qmlTazxtPpa5LJSi98U4VSqzz5AUAiIfmBQAIh+YFAAiH\n5gUACIfABoCKUuECLxyg9olSvPFSKtygxjMNDAxknSclfa0qHFHIyCR1/d41qcBFIddfqlJHQRWL\nJy8AQDg0LwBAODQvAEA4NC8AQDgENgCMON7UBlVXgQFvmoUKMqj1hey9pfa5yt2jy6OuX00H8c6v\n7pMXglHXdSJDGMWeiycvAEA4NC8AQDg0LwBAODQvAEA4NC8AQDikDQGEoRJzKq3mpRW9xF2uvr6+\nrHN546kUdf2FJCBz9wnz7kmlxjv9rNj9wHjyAgCEQ/MCAIRD8wIAhEPzAgCEM67Yl2UAAFQKT14A\ngHBoXgCAcGheAIBwaF4AgHBoXgCAcGheAIBwaF4AgHBoXgCAcGheAIBwaF4AgHBoXgCAcGheAIBw\naF4AgHBoXgCAcGheAIBwaF4AgHBoXgCAcGheAIBwaF4AgHBoXgCAcGheAIBwaF4AgHBoXgCAcGhe\nAIBwaF4AgHBoXgCAcP4LkMpAZYOBNGwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7adbc7a20>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_stacked_denoising.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing the extracted features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_stacked_denoising.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_stacked_denoising.ckpt\")\n",
    "    W1_val = W1.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure extracted_features_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAABWCAYAAACaXQIdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztXVtsXFfV/s6Mx+OZ8d2xx3YuThzbTQhJm0sJhbRAKFBS\n0UoIJEB9gvJQqIAnEPCChCokeEBCggcgFCQQoqFclHIpfShNgaZAXdJb0sZ1WjtOHMd3j8fj8Vz+\nh6NvnTVnpv2bIuGddH0vHs+cOXPOOmuv/a3LXtsrl8swGAwGg8E1RNb7AgwGg8FgqAWboAwGg8Hg\nJGyCMhgMBoOTsAnKYDAYDE7CJiiDwWAwOAmboAwGg8HgJGyCMhgMBoOTsAnKYDAYDE7CJiiDwWAw\nOAmboAwGg8HgJOrW40d/8IMflAHA8zx4nlfxWa3WS3yvVCqhrq5OXvMcPEa/BoBIJFJ1DqJUKiEa\njVZ8po8JX1epVJLz3XPPPZUfOoBvfOMbZcC/zmKxCAAoFAoAgIaGBgD+/fEeeH+e52FtbQ0AEIvF\nKr7neZ6ci7KJRqNV8tVy5vF8LxaLyfnDz6Curg65XA4AcN999zkl07vvvrsM+NfM66Zc6uvrwc/C\neuJ5nugmZaD/p3zC39Pv8XvlcrlKnvoZ8rz8W1dXJ8f/5Cc/cUqe3/ve92TM8/rz+TwAyJgGqnVE\ny4mf6XH+emM4jNca3+HzUob62u69916n5Hn06NGqGw3fx2shbCdrfRY+52v9Tvhc2k6+Hj7zmc+8\nIXmaB2UwGAwGJ7EuHlQtJhNmQJoNavau2SIAYeee5wmzJTMrl8viFfB3VldXAfjMPsxmPc+rYiFh\nT81VUB7FYlHugZ5T+F4AVHgFyWQSQCAb7SHxNT9rbGyU81B+ZLGFQqHiOgD/WYSfcSqVAgAsLS3J\nd12Dft58Tf0itDdDFAoFeW9lZUWOAyo9TeolnxFQW558XevZEdordl1PtUdKXanF/PX9EtqzAfz7\nDntQhUJBZMvnRS99bW1Njn89r43H6LF0NaCW91PLw9fRDSAY24VCQY7X4zKsv9omhu12Le/p9SJT\n/x/WZYLSN1or9AZUDnStcBQcP6MSxuNxxONxAIHAAaC1tRUA0NzcXPG9bDaLc+fOAQDm5+flN/lb\n4UGjH4aL4KSsjRRDUhzYqVRKZKPDSTSkiUQCAJDJZAAAs7OzIhsO6OHhYUxPTwMA3vOe9wAAOjs7\nAQBdXV1iDPiba2trFSFDfX6XDerrERMtO62v/F42mwUQ6BXvd2lpSb7b398PAOju7pbPn3nmGQAB\n6erv78eGDRvkt4DahlWHplzV0VoyC3+mJ3z+LRQKotuLi4sAAj1ta2tDV1dXxbmKxaJ8l7ZiamoK\nADA9PS2ftbS0AKi0KbVI8tUIrQPhcHM0GhWbST1bXl4G4MuJukgSOTQ0JDaRes3nsbq6KjLSxJSy\n1c4D/16pflqIz2AwGAxOYl0pQiQSqUpMks1oV7SxsREA0N7ejra2NgAQb0mzU7539uxZAMClS5fw\ntre9DQCQTqcBAB0dHQCA06dPy3GTk5MAfNZAj4thL4ZgSqVSVYjHJdRi+mQ+mt3wHsis4vG4yJfM\nlB5VPp8XxjM2NgbA95b27NkDANiyZQsAYOPGjQCAnp4eYav83uXLl7G0tCTn03+j0ajI2TVoeVI3\nKc9aIQ7e08rKCi5fvgwg0Ct6Uk1NTejr6wMAXH/99QCAgYEBkW24GCOVSol8yEZLpVJVMYUupKgV\nGnMBvLdisShjnPdUyzusFemYmZkBEERNBgcH5Th6mk1NTTLG+UzOnz8PwH82OpkP+DYl7DnpMRIO\nK7qI1wqzlctluReO8VgsJuN8YWEBgG8nAd+Dolwow3K5LO/xudGTzWaz8luMUDU1NclzJWoVsr1R\nmAdlMBgMBiexrh5UreSZzptwViYLaG9vF4/m4sWLAICXXnoJgD/78xx8LxqNCtsnO2MctVwuCxMY\nHh4G4M/+O3bsAODnBvT3otFoFTNwEZqhhBl2LpcTWfK+Ojo60NTUBCCQ28TEBADg3LlzwqR27doF\nwGdi9AL4Hj2E2dlZ9Pb2AgjygPSegMCj0NdID9hlvJFCDp3n4/GMxTMvd+bMGZEtWf358+clB0Dv\nnTJMp9MiH3oRKysrosNkwtqzeyMlvuuNWnkRwPfm6TFSR9bW1uQ1PxsYGAAAbN++Hf/+978BAA88\n8AAAYP/+/fjIRz4CIMitDA0NAQAOHDgg+qijLJQf5Ul4nud8HqqWV0K7mkgkxIYyT9zW1iZyn52d\nBRA8h82bN4uMGY1aXl4WOXKc0zPN5XJiJ2mXo9GoRLmou7Tp+treKNzXZoPBYDC8JbGuZea1qk3I\nDqempiTmTGbZ0NAgsznj/GSbN954o8RUGdNvbGyUGZ7eAT2vTCYjjIl/dcy/vb0dQOBpaBbgImrF\neclMKbN8Pi9Mh7m5bdu2YXR0FEDgOV24cAEAMDc3J3Jg5d7o6KjIlx7lvn37APgyO3HiBIDKfBZl\nzvwUK4R05aWrqBUzJ+NMJpOiH4zxZ7NZyc1RdmSU+XxevEh6n3v37hUPi3KhnE6cOCFjgFV/o6Oj\n8iy018ZrdbVsXyPMoqmTPT09Mv443l599VV578CBAwCAgwcPAvD1b2RkBECQM9mwYYMw/bm5OQCB\nN9Dd3S3nZZ61tbW1qiJYl5m7mtMjaukn76OtrU10g/pZV1cn45eRI3qVO3fulHPwmPr6esnfU+/0\nsgbaAH5X5/R4PM9PmV8JnCmS4MT0wgsvAAAeeeQRMWxUvn379slkxUHPgb60tCSGlkoVi8XE2NJI\nc8CvrKygp6cHAHDbbbcB8BVz06ZNAIIwgjbarpZEa+h1JjR8vO7m5mbs3r0bgG8YAX+ifvzxxwH4\nMgeAd77znQCArVu3SviPxrCjo0NKeznpkxh4noff/OY3AAL59fX1SSjhxRdflPcA3+i6KtNweFS/\np4tlqGvUq1KpJLpJXeVAff7550VHv/vd7wLwdZQTGeXOCf++++7D1q1bAQB33XUXAL+Un/JmGFUv\nx3A9qa+XFoTXl+XzeQmL0qglEgnRJcqH+jczMyP6+fa3vx2AbxdojGk8WUDheR7OnDkDIBjXAwMD\nYoA5edEWuSzLWt0cSJaoT6lUSnTvscceA+CHNGkLqT9MaywtLVXZjkKhIBM80yWU64ULF6rkn0wm\nxXngmLAiCYPBYDBcc1jXhbqa7XNWP3XqFAA/zEEmdMcddwDw2TvDJ2RVDIfMzc3JOehubtiwQdhT\nOBxSKpXk/CyTvnTpkrAKHRYDfHbqcpiPTL5cLleFMXgv7e3t2L59O4CAoX7rW9/Cgw8+CAD4/Oc/\nDwB4xzveAQAYHx/HP/7xDwBBmf7evXvl9XPPPQcgKCgZGxuTMAyP2bp1qzAqPiud1Hc1hKIXF4dX\n0OvEOWXM96anp0XeTBIzIZ/JZOS8n/jEJwD4bP35558HEDBOymnPnj3iLfFcGzZskPfomVKexWLR\nWdZfq/edLn0G/OunXlIGjY2N4olSTpR5PB6X75LVZ7NZvPLKKwBQ5f1PTEzgD3/4A4AgjN3d3S36\ny2fD8HQul3M2ZKq9UOoeCyHohU5PT+NXv/oVAODYsWMA/KUh733vewFAvPNt27YB8J8Dw6IsIItG\no/jnP/8JIJAnf093gqGdTSaTosfUWY6HN1PEYx6UwWAwGJzEunhQtbovM+5LD+fw4cMSc2bsc3Fx\nUfIXzEvp8tsbbrgBQMAuEomEnJcsjKxtfHxcWBpjrNFoVI7TveR4za62kQECr3R1dVWYIO+PHkwy\nmRQPkXmnl19+GR/72McABM/gO9/5DgCfUdLjYtK0oaEBP//5zwEESejPfvazAIBbb71VvLFDhw4B\n8BPfZK2Ma+su3K7KtJaXT93R5dHhRaa6tyHvk9+bnZ0VeVLP77zzTpHLk08+CSDI0cViMclZ8a8u\nu9ZMn9fjak5PLyKlfur2XICvrxx3zEt5noff/va3AIJxyjLyZDIpBRb0AjzPE9tAJk/Pi0weCIqr\n8vm86D3PX2shtsugHtCLoUe4sLAg9059O3LkiMiPOX564ul0Ws7BZ9TU1CReLc+hlwK9/PLLFefq\n7+8XG6PzWID/7K/Ug1qXCUpvmUEXnTfPooehoSHcdNNNAILBee7cOQnxhUN2ujEsDfPMzIyE72gk\nbr75ZgDApk2bZA0FBbllyxZRbiqpXjXtMhjiiMfjEgLiNVOJdu/eLYUNNIb33HOPDPJvf/vbAAKF\nP3LkiJzj8OHDAHzZ/uIXvwAQJKtZubd3716poDp58iQAv2MHB8Cjjz4KAPjoRz8q1+1qFZ/uzsDX\nDFlwAspmsxLi4KDUnR6YoKasU6kUNm/eDCDQ8927d0vI6l//+heAoIrsgx/8IB5++GEAwBNPPAHA\nH+wMYYf7UurGvK5Ch3mos7p/I++JchoZGRHZMoRFLCwsSFEV7zudTsukwoIIjvM9e/ZI9SrtxuDg\noExc4THucgia0PKk7CjP+fl5sXt33nknAL8gjMSAsmNIcPv27TL2ecz8/DwGBwcBBCSf8lpaWhL5\nsBtFd3e32ADqNeXa3Nxs66AMBoPBcG1gXTwovVEZX5PFX3fddQB8Nk6myjDe5OSkuIsMW+nN8Mhw\nX331VQA+m+KsT8bK8EmxWJQkP5nr0NCQhKbCK9qLxaLT7JSeiO7ITqZDhrSwsICnn3664r1SqYTx\n8fGK977whS8A8GXGBD8LJ1paWoSFfu1rXwMA3H777QCAhx56SNgcvYdLly5JQppreXitiUTC2cIT\n3RNSdzUAAt3T/Rl1qJp6GGaQAwMDwjTJPI8cOSKMl2AJdDwel+/yWerSfJaz02PWm3C6hlqbAeqN\nHwH/Hrn0g+OvWCzK0ohbbrml4lz3338/jh8/DiDQQc/zRJ8ZSaE3tmXLFtFdev+rq6v4z3/+AyAI\nWTOKondUcA26dJs2lNevPXLKkUsXurq6ZLkHz8GQYGdnp9haRkXi8bh04mBXCupuKpXCs88+CyAI\np87OzlbsZMBr5O9ZmbnBYDAYrgmsew6KYEyYnlQsFqvq/bRz5075DldB8xjNLMnC0um0FE6wpFIv\nCiTT1aXtZAtkWHr1tKtsHwgSkXV1dSJD5ut05wbmMsgqZ2ZmhGkyQUoWNTU1Jd9lHmBiYkJi+5/6\n1Kcqju/q6hKWe/r0aQC+18tFfGR2ZP7ZbLZiwz6XoHtDvlaMPxqNClukF/Pcc8/J/YUXPW7cuFHy\nKPTel5aWRFbMtdLrX1lZEVlTLxsaGsQLC3v54dcuQW86Gu4AT0QiEcmZ8B4HBwdFR6iL1OHHHntM\nvCtGPoaHh8XD5WJ0Ih6Py/IH/r18+bLYAf4mf6eurs75HFS5XJaxSf2hTlLHgGBsZ7NZibBQPrSJ\nly5dkhJ9Hr9jxw7RR3rz1OeFhQV5NrqPod4AEajclPNK5WkelMFgMBicxLq36g3vT6Qre1j9RU9g\naGhIepnxL1sTeZ4ncVB6VblcTqrWmIPiQtKNGzdKvovnHx8fl3LJcF5AV2e5CN32hjIlq6G3lMlk\npCqRJfy7d++WKjN+Rg90586dEp9m/PnYsWP4/e9/DwDinbLS7Ctf+Yocx27d+/fvF++VuUGWrbqa\nLwEC9qdLt/keGajnecIuqXvXXXedyJv5uw996EMAfC+e+U7q6I9//OOqnUfZqubkyZPymow/l8uJ\nJxpmo3V1dc6XRZfL5Yr8HhDcRzQaFR3RbZCYE+Ki8V//+tcAfF3/5Cc/CSBYnH/27Fk5np47IwTJ\nZLIqkjI5OSl6rxeU6r8uw/O8Kn1gzndxcVFybpTnxMSEvNZ5aMD3oGgz6LE3NjaKF0/PklhbW5N8\nMr225eXl19zn682U7a9rs1ggmJDCbe3n5+fFINBI5nI5cTOZcOfq5vr6ejmX3iiO7juTeU899RQA\n4N5778X+/fsBBMb9j3/8oxzPEB8F7/rAp2Ktrq7Kaw5UGoRSqVS12eCTTz4pCk45syCis7NTBjIn\noRdeeAF33303AOCvf/0rgCB5nUgkZPJi769Pf/rTMhAYvqGMM5mMsyE+PXmGnz3lG41GRW85ePv6\n+mTyYUEKB/vf//530U2uw3n22Wfl/CRblFexWKzqbrCysiIGlSE+kgLXl0IAtTtKUIblclnko7uN\nMPTGsmgev2vXLtFxfq+xsVFIKMNUDL92d3fLhE8dHBkZETIVnjBLpZKz223U2qqI9o/329bWJiFl\n3dWBIWgeT8KYyWSqtmt/4okn5Hwk9DxnJBKR6yAZi0ajVQVsurjCiiQMBoPBcE1g3T0ovRU0gIre\nY2Q7ZLOnTp2SlfgMo3AR6tramjCDW2+9FYCfnKML+pe//AVAkDg8evQovvjFLwIItp7IZrPi5pPt\nk+HrDg0uo6GhQTzCMCNsamqSRXfccvynP/2pMNPPfe5zAIAf/ehHACoXkR49ehSAv60JPScyVLKz\n48eP45e//CWAoCBifn5eyl8ZNiWzamhocDbMp5me7iwAVG7NTr2lLC5evCgeDr1P7fHQg2Xng0Qi\nIc+L5+Ui57W1NQmT0CtbWFiQMDQXoevN4lwNS+lNCsNbk3Pcnj9/XkrsydLb29sljPrhD38YQBCu\nX15elpJy6tbFixclxMXiHf6fSqXEG2Mo8YEHHpAQFkOxOmzlauREl27Ty+NfHdqnDeUYjEQi4jFR\n76inra2tos/0zltbWyWaRB3n9y9cuCDjnCHuU6dOyTl4Hboo7kp7RZoHZTAYDAYnse7dzIlwO5mu\nri7JL5H1LC0tyXc5c5MJ5XI5yZ3wmO7ubmFW9KroIf3ud7+TfVCYi2ppaRFWqkuzAffbyPA+E4lE\nRUwfqFwYzc/Y/qWnp0fyclywyPj/2NiYlECzf9fBgwcln0emz9i053nCyphjmZiYkAR/OEeiezG6\nhloxfr2VNuDrI70jetednZ3COKlf9HhaWlrwgQ98AEAgi1OnTlW1nKFHMTc3V5Gv43v0mMhy9dIJ\nV6FZNOXI+9D7WfHeuaC+XC7LfbLAQesRmTtl19zcLPkQnlcvAWDE5c9//jMAv/3W+973PgBBVEaP\neVc9KMpTNxDge7SbTU1N4kExz5bP56uKKnTfw3A7qY0bN4pe6ZZagK9/HOd8RrlcTq5DbzQL+OPl\nSm3oumYA9cUy1MObz+fzIhDdA421+5xoOLgLhYIIntV/Z86cEeVkeICJ6K6uLklia0Ouw4NA8LAX\nFxedDvFxUqmvr6+QIRDIL5FIiJvOgRqLxfDVr34VQDCpsEV/uVyWRrBMdCaTSdx///0AULVx4Z49\ne/Dxj38cQDBhDgwMyMDnc9GNVV1dqU+d0MU34SqmqakpkTvvN5lMSviOusr7b25uFh1i9dnMzIwY\nSBpnrr3r7e0Vneaz7OjokHETvh5dIecawmE9IJgIdBNS6iBJwOTkpIx1vqd3jKZe6tAUxy7D2Qzx\nTUxMyDofhkxvueUW2fqEoX69A7Sr+qmJHeXB+6YeZTIZmcD11iPUWY5DkvzFxUV5FnoXXMqKsuaE\nduDAASEGJAVtbW1V+qmrNC3EZzAYDIZrAuvqQdVy98kYo9GouJtk+zrhrhku4CdTyZj42blz52T2\nZyiG5dU33XSTsF52pejp6ZGyVYaq6Lq6zE6BgJnmcjlhMJQlPYBSqSRslCy/q6tLZMRtNMh8zp49\nK2zytttuA+D326McRkZGAARrdLq7u4Xl8pndfvvtcj08F0uCi8Wis0l9Ym1tTRgn75v61draKl6A\n7hzP17w3fcxDDz0EIFju0NbWJmXRLBDgWJiYmJBQl+7/Rx3lcbqXoqshU0J336auMBTE+wKCcZfP\n56u2cKc3OTMzI0yf3kA+nxePlV6D9tQYQeHf97///eI58Vz0Oni9rkNvVqpRKBTE7lEmvb29Yld5\nPOUai8VEVrQJsVisorMJEMg/lUrJa104FPbC9KavVwrzoAwGg8HgJNY9B6UTpECQB2lsbBQmytxF\nJpMRtkhPgBvpbdmyRbwCxkrT6bTM+mTvugiD5aWMS6dSKWEcnPX523qjOBdRK5nLe2ecGAg8Q7Lv\n06dPVxVTvPvd7wbgs1iyK277XF9fL0UrXIDKrhT79u3DD3/4QwCBh9bZ2Sl5QP62TrK6mtjXZdFE\nuBdfqVTCwMAAgECenudJZw7uf8Vikp6eHtHDu+66S87L50TPkt0mtm3bJl4Gr0d3AiDLpefhuo4C\ntcc883KZTKZqU7zBwUGRD/WICf+5uTlZdsLISj6fl036eDw/q6urkyIodkHZvHmz/Bb1mse7vKGm\nRrhIQnc3oa5ShtPT0yJ35qN1oQPvl8fr19Q3ynBqakp+m3m+rq4u8fbpzbMBgHUzNxgMBsM1g3Vd\nqFsqlari0Iz/Li0tibekt2QneDz7xq2trQkDpXcVjUalIoo5EXpXTU1Nch1ktTMzM7K4j9dBFlZf\nX+90FR/ZSiwWE0+FjIrx/EKhIGXj/CyTycgiZnpOBw8eBOB7PFx4y/1kIpGIMCgyKsb6x8fHhUmx\nvPXEiRNSvk5mpXuAudpKhkzP87yK3XWBgPk3NTXJZ/SkUqmUVFHR+yYjj0ajUpJPvWpsbBQPi7pK\nj2jr1q2SH6HujY6OVuyeDAQ5BMD9llz6+igf9sq7cOGCyJF6MTc3JzKmN0Bv9YYbbpDPtCwYeaEu\n0lPo6+sTb4F46aWXpAs4r4fPWUcQXAbtIr1Pjq9cLldVKbm4uCi2jXkp6msymZRnwfxRJBKR84U9\ntEgkImOaOptKpeT8jEbRPjQ3N18dZea65DS8ZoeG7dKlSxUTDeC7iuHebQzZra6uSpKfg3p1dbXK\n/WVp9OzsrExaujEly9Zr9QlzOWFKGRWLxapmjTSojY2Nkoin8gwODopCcX3TsWPHAPhKygIIGtan\nnnpKZMjwHA1sJBKR8ArP/6c//UnK0LmORYcQXTWoep1JGJT19PS06CuJTW9vL+644w4AwYT8+OOP\nAwAeeeQR6eDB0NTx48dlwHM9Hg1HR0eHGBb+ZktLixjSsLEF3N9uQ4dMOSHrNV4MJ3Hsz8/PV3VG\nIUHatGmTTC48r57Q2ECWTWM9z5NJ7plnngHgFwKRaLEYQ/f+dLkwCqjsb0ciqtfHhfty6tSJ3iiS\nx/O7elNREizqoi6C4DhhMc/4+LjImBPbfyNDC/EZDAaDwUmsayeJUqlUlTjXK5nJhPh3fn4ef/vb\n3wBAepoxQX/mzBkpmGDIrqmpSZg8mQTZ/MjIiIRZ6HH19/dX9InTf3O53Jsqk/xfIxKJVIQogMqV\n3PzsXe96FwB/ozcycnpSTBY3NDTIwmV6Xps3b5YSXXoNfCbJZFI8LYZbh4eHhVFx4aou13Y1hKLD\nerqzNlBZ7MGQCI8ZHR2VMDI9qJ/97GdyPEvKyS77+vpk0z2yUPaLvHz5shSikCWn0+mKLd6Bys0q\nXfVI9ZYlvO5w8l3bA45Xvdi0Vnk99YxhqnQ6LTJgoQ6PP3/+vMiO46ChoUGWqlAv+dfV8LOG53li\nO3U6AvC9II5bekE7duwQT5HHU4c9zxPvnfLRHjn1jJ7vyspKVbeIeDxe0QVEX08kErEiCYPBYDBc\nG1j3HBSZarhPWzwelxg7mdbo6KjM+mTenK3X1tbEYyBzHRsbq1hUpr+3uroquRAyrZ6eHrkevXEi\nULnA0GUUCgVhgLwHepb19fUSW9YtZb7+9a8DCMqb2fpk06ZNwsAo0+Hh4aqFfmRI/f39uPHGGwEE\njK2zs1OOI8vVvb94Pa6iWCxWlcKT8esEMnUjEolIST6Tw1/60pcAAIcPH8bhw4cBBAn5Q4cOyeJd\ntt5iXiWdTksREOXU1tYmz5dy1Ezf5Twp4F+r3gwSCHIguiCF4/XAgQMVfeSAQK59fX2SAySDX1hY\nkLY7XIBPZDIZyfPpjU55Po51LU9Xc3qE53lV5focqzoHxeKoRCIhXjy/R/taq/XY5OSk5AjDC9aZ\nnwaCgpTV1dWKtkpA5f5aV0WRBFEul+WCqXw6XEHjyM0JDx48KEKiQlKRd+zYIeuaaGALhYIMYoZd\niGw2K4aTLi9dVyBQVi1Q11fpA77yhJVMF3pwEqeM2tvbxUBQpiyC+P73vy/VPOwa8eUvf7mqMODm\nm28G4K/K53c5IHK5nJCK8LYAQBAucxV6F9Bwk01OMkBQ2LCwsCBhJz1JEzSa1NWHH35YQqo8juSh\nublZzkV9jEQiYnQ4FvRuyq4b1HK5XBFeA4KihFQqJfqptyOhbaCciNnZWXkWtAPLy8vyXDhe9e+E\n5dPZ2Sk6Hg71uRoufS2EN1fVzXepW4VCQXSJtlYTTaZEqFPZbFZky8mdodapqSlZQ0oZxuNxuQ6e\nX1fEXqlM3XcJDAaDwfCWhHMbFhKlUkm8nhdffBGAP5vTlWTYin/T6TQOHToEIFiz09PTU1Wmq9c8\nsWBC95jSnZLD1+wyO9VJTcpUbxsAVPbC4mdTU1PC9Mk0Wcb7zW9+U7wkhg12794tSXzKiPLW22cw\nvFgsFqu6IejeX64mojXrIyhX6tTKyoqweuqVLgKgV0Vm/uijj8o56JGOjIxUFd+QvbJvJFAZXtFL\nCsLX7GqIT699DCfg9TYjjGYwca9Dgrxf2oNCoSA2glGWlpYWKeGnHOmFrqysVG1fsrq6Ks8nXM5+\ntXpQvP7p6WlZA8aoBxDIgxEjjvF4PC7Phsdff/31Ilt6YbSX2WxWwt06bKvHt76e8Os3AvOgDAaD\nweAk1oW+1mIoYcYaiURkVmYBBVkPgCpmkEwmhT1ysejTTz8tDJ3MTCcG9b5EgM/Wwj3YwnFd16EZ\nKj0nXTRB+TIOfeHChaqYPZnSrl27hF3SUzh58qTkSOhJsAw4kUhI9w/dzyzM9HWRxNUg11qLtgE/\nWU9PVPc8YwKeuskF4a+88orkBIh8Pi9yDK/U1xvmcQxEo9GqjiY6Z+J6IY++vvC1RiIR8WJY6DA2\nNiZFIyy558Lbvr4+kR1llkwmRffopTLHmsvlJMeivVaOeY4N7YVeTfpJ6Jxk2OuPxWJVnSeY70sm\nk+IRMeob/OTXAAACwUlEQVTR2Ngo74UbAOiiB55TFz7Vkp3loAwGg8FwTWDdPajwDKw9FrJSnVNh\nBQlLGTU469Mj0DM8Z3+9RxLPW8uTC++x8mY68f4vwevUu1aGY9LRaLSqkq5QKEjuifLT3Y3JZPWC\nVDJSslZ+Njc3J+cl6yoWi1VMWcvYVcavnzt1M+y5xGIxkR29znQ6LeW34SqyXC4nXgAjAPF4vOLZ\nAUHJbqlUqvpN7QWHn68+h2vQrJuvw+XmOoem8xisDKVnxLxzoVAQT4i6qBeu0n7Qo9C7ydYa83pb\nesJVeWobGm4jxWvW90FdWV5eFn0MR4vy+bzITO+jR5vL9+h5zc/PS9WlXtge9kj1WLpSD2pdiyT0\nBYdDfHpNhN7OnO+xdFlDhwCBSmPNz2oNFKKWAGslyV2EDklSQTlQw+sR9Hv6nlgqyvL+iYkJOY4d\nIpaWlqrWQ2jZ1jpvuK+dXhfh6gRFaKMWLsuNRqNVIdOxsTFJOofLbDds2CChOq7ticViVURJ/87r\n6Rxlp8eQq0UShA7dhycEvSkow8C9vb0SMg3rFokSEPR+1E2dw502dMiLqCWzq2FpyRuxRXqy1jKg\nrlKv+b+2Hbq4jMUU/Ex3niD0OA4T+f+G2LttHQwGg8HwloXnsldgMBgMhrcuzIMyGAwGg5OwCcpg\nMBgMTsImKIPBYDA4CZugDAaDweAkbIIyGAwGg5OwCcpgMBgMTsImKIPBYDA4CZugDAaDweAkbIIy\nGAwGg5OwCcpgMBgMTsImKIPBYDA4CZugDAaDweAkbIIyGAwGg5OwCcpgMBgMTsImKIPBYDA4CZug\nDAaDweAkbIIyGAwGg5OwCcpgMBgMTsImKIPBYDA4CZugDAaDweAkbIIyGAwGg5OwCcpgMBgMTsIm\nKIPBYDA4CZugDAaDweAkbIIyGAwGg5OwCcpgMBgMTuL/AECsRWd/T9yxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7afd39f98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    plt.subplot(1, 5, i+1)\n",
    "    plot_image(W1_val.T[i])\n",
    "\n",
    "save_fig(\"extracted_features_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sparse Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\anaconda3\\envs\\tf_new\\lib\\site-packages\\ipykernel\\__main__.py:3: RuntimeWarning: divide by zero encountered in true_divide\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure sparsity_loss_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcTfX/wPHX274MSlR2kqEoYqSU0pdIkZAUpZKo+Cah\nEknafpUlSUVZWoRsiUpU9n34RhQjkexblrGNMZ/fH+9rGmOMmTFzz13ez8fjPmbm3nPPec8xzvt+\nPufzeX/EOYcxxhgTaLJ5HYAxxhiTEktQxhhjApIlKGOMMQHJEpQxxpiAZAnKGGNMQLIEZYwxJiBZ\ngjLGGBOQLEEZY4wJSJagjDHGBKQcXgeQFYoUKeLKli3rdRjGmADkHKxeDfnyQYUKXkcT2lasWLHX\nOVc0o+8PyQRVtmxZoqOjvQ7DGBOAPvsMHn4YJkyABg28jia0ichfF/J+6+IzxoQN52DQIKhcGW6/\n3etozPmEZAvKGGNSMncu/PILfPwxiHgdjTkfa0EZY8LGoEFQpAi0aeN1JCYtwqYFdfLkSbZu3crx\n48e9DsWkIk+ePJQsWZKcOXN6HYoJMX/8AdOmQe/ekDev19GYtAibBLV161YKFChA2bJlEWvbByTn\nHPv27WPr1q2UK1fO63BMiBk8GHLkgCef9DoSk1Zh08V3/PhxLrnkEktOAUxEuOSSS6yVazLdP//A\nqFHwwANQrJjX0Zi0CpsEBVhyCgL2b2SywocfwpEj0LWr15GY9AirBGWMCT/Hjmn3XsOGUK2a19GY\n9LAE5UcRERGJ33/33XdERkby119/0bdvX/r375/m/YwePZrOnTsD8NFHH/HZZ59leqzGhIpPP4Xd\nu+H5572OxKRX2AySCCQ//fQTTz/9ND/88ANlypS5oH098cQTmRJTfHw8OXLYn4MJLadOQf/+ULMm\n1K3rdTQmvcLyivTMMzpZLzNVqwbvvnv+7ebNm8fjjz/Od999R/ny5dO8/1GjRvHmm29SrFgxIiMj\nyZ07NwB9+/YlIiKCxo0b07ZtW5YtWwbA5s2badKkCb/++isrVqzg2WefJTY2liJFijB69GiKFStG\n3bp1qV27NgsXLuTuu+/mnnvuoU2bNpw6dYpGjRoxcOBAYmNjAXjnnXf46quvOHHiBM2aNeOVV15h\n8+bNNGrUiJtvvplFixZRokQJpk6dSt68efnjjz944okn2LNnD9mzZ2fChAmUL18+xf0Yk1UmTYKN\nG+Gtt2xibjCyLj4/OnHiBPfccw9ff/01lSpVSvP7duzYwcsvv8zChQuZNWsWv/3221nbVKpUibi4\nODZt2gTA+PHjadWqFSdPnuS///0vEydOZMWKFbRr145evXolvu/AgQPMnTuXbt260aVLF7p06cLy\n5cspXrx44jYzZ85kw4YNLFu2jF9++YUVK1Ywb948ADZs2ECnTp1Yu3YtF110EZMmTQKgTZs2dOrU\niVWrVrFo0SKKFSuW6n6MyWzOaWKqUAHuucfraExGhGULKi0tnayQM2dOateuzYgRIxg8eHCa37d0\n6VLq1q1L0aJaFLhVq1bExMSctd19993H+PHjeeGFFxg/fjzjx49n/fr1rFmzhtt9hcdOnTpFsSTj\nbFu1apX4/eLFi/n6668BaN26Nd27dwc0Qc2cOZPrrrsOgNjYWDZs2EDp0qUpV64c1Xx3nmvUqMHm\nzZs5fPgw27Zto1mzZoBOvk1tP7fcckuaz4UxafXTT7ByJQwfDtmzex2NyYiwTFBeyZYtG1999RX1\n6tXjjTfe4MUXX8zU/bdq1YqWLVvSvHlzRIQKFSrw66+/UrlyZRYvXpzie/Lnz3/e/Trn6NmzJx07\ndjzj+c2bNyd2NQJkz56dY8eOpXs/xmSF//s/uPxyeOghryMxGWVdfH6WL18+vv32W8aMGcOIESPS\n9J5atWoxd+5c9u3bx8mTJ5kwYUKK25UvX57s2bPz6quvJraMKlasyJ49exIT1MmTJ1m7dm2K77/h\nhhsSu+jGjRuX+HzDhg0ZOXJk4v2obdu2sXv37nPGW6BAAUqWLJnYGjtx4gRHjx5N936MyagVK7QF\n1bUr+BrwJghZC8oDhQsXZsaMGdxyyy2J3XavvfYa7ybpe9y6dWvi98WKFaNv377ceOONFCtWjOrV\nq3Pq1KkU992qVSt69OiReC8qV65cTJw4kaeffpqDBw8SHx/PM888Q+XKlc9677vvvsuDDz7IgAED\nuOuuuyhUqBAADRo04Pfff+fGG28EdLj8F198QfZU+k0+//xzOnbsSJ8+fciZMycTJkw4534uvfTS\n9Jw+Y87rrbegYEGwxnpwE+ec1zFkuqioKJd8wcLff/+dq666yqOIgsPRo0fJmzcvIsK4ceMYO3Ys\nU6dO9Xsc9m9lLsSGDVCpEvTood18xjsissI5F5XR91sLyiRasWIFnTt3xjnHRRddxMiRI70OyZh0\ne+MNyJVLp5OY4GYJyiSqU6cOq1at8joMYzJs0yb4/HPo3FkHSJjgZoMkjDEh4803dUh5jx5eR2Iy\ngyUoY0xI2LIFRo+G9u2hRAmvozGZwRKUMSYkvPWWfrWisKHDEpQxJuht2waffAKPPAKlS3sdjcks\nlqD8SER48MEHE3+Oj4+naNGiNG7cGIBdu3bRuHFjqlatytVXX82dd94JaMWGvHnzUq1atcSHLbFh\nzL/eeUcrl/fs6XUkJjPZKD4/yp8/P2vWrOHYsWPkzZuXWbNmUSJJZ3mfPn24/fbb6dKlCwCrV69O\nfK18+fL8ktkl2I0JATt3wrBh0LYtlCvndTQmM4VvgkppcZjGjcFXIDXdr8+Zk6bD3nnnnXz77bfc\ne++9jB07lgceeID58+cDWrW8QYMGidtee+21adqnMeGsf3+Ii4NMLm1pAoB18fnZ/fffz7hx4zh+\n/DirV6+mVq1aia916tSJxx57jNtuu43XX3+d7du3J762cePGM7r4Tic1Y8LZ7t3w4YfQujVceaXX\n0ZjMFr4tqPO1eC709XO49tpr2bx5M2PHjk28x3Raw4YN+fPPP5kxYwbff/891113HWvWrAGsi8+Y\nlLz1Fhw/Dr17ex2JyQrWgvLA3XffTffu3XnggQfOeq1w4cK0bt2azz//nJo1a9qCfsacw7ZtMHSo\n3nuqWNHraExWsATlgXbt2vHyyy9zzTXXnPH8zz//zNGjRwE4fPgwGzdupLSNmTUmRa+/DgkJ0KeP\n15GYrOJJghKRwiIyRUSOiMhfItL6HNvlFpGPRGSXiOwXkWkiEvRzxEuWLMnTTz991vMrVqwgKiqK\na6+9lhtvvJH27dtTs2ZN4Ox7UO+9956/wzYmYGzaBB9/rFUjbORegFq58oJ34clyGyIyFk2OjwHV\ngG+B2s65tcm2ew5oAzQADgLDgQjnXPPU9m/LbQQ3+7cy5/PoozBuHPzxh5U1Ckjz50O9esjJkxe0\n3IbfW1Aikh9oAbzknIt1zi0AvgFSWpi5HPCDc26Xc+44MB44e6U9Y0zYWL8ePvsMnnrKklNA2rgR\nmjXLlKatF118kUC8cy4myXOrSDnxjABuEpHiIpIPbU19n9JORaSDiESLSPSePXsyPWhjTGB4+WXI\nm9dq7gWkgwehSRO9OTh9+gXvzosEFQEcSvbcQaBACttuAP4GtvnecxXQL6WdOueGO+einHNRp5dR\nT2GbjMZs/MT+jUxqVq+G8eN1McJLL/U6GnOWv/6Cw4dh8mSoUOGCd+dFgooFCiZ7riBwOIVthwK5\ngUuA/MBkztGCOp88efKwb98+uwAGMOcc+/btI0+ePF6HYgLUSy9BoULQrZvXkZgUXXstbNiQciWe\nDPBiom4MkENEKjjnNvieqwqsTWHbakAv59x+ABEZAvQTkSLOub3pOWjJkiXZunUr1v0X2PLkyUPJ\nkiW9DsMEoCVL4Jtv4NVX4eKLvY7GnOH992HHDv3HycQPmH5PUM65IyIyGU007dEk1BSoncLmy4G2\nIjIHOAo8BWxPb3ICyJkzJ+VsPKoxQck5eO45uOwy7d4zAWTGDOjSRWuVZnIPlVcTdZ8C8gK7gbHA\nk865tSJSR0Rik2zXHTiO3ovaA9wJNPN3sMYYb02friOX+/aFiAivozGJ1qyB++7Trr0xYyB79kzd\nvSfzoLJaSvOgjDHBKT5er3+nTun1MGdOryMygK5zcsMNcOIELF8OKXTNi8gFzYMK32KxxpigMHo0\n/P47TJpkySmgzJsH+/fD7NkpJqfMYC0oY0zAOnpURyuXKQMLF4KI1xGZM+zdC0WKnPPlC21BWbFY\nY0zAevdd2L4d3n7bklPAePFF+N432yeV5JQZLEEZYwLS3r263tPdd8PNN3sdjQHgvffgzTfhxx/9\ncjhLUMaYgPTaaxAbq9dDEwCmTtUx/vfco01aP7AEZYwJOBs3wgcfwGOPwdVXex2NIToaWreGqKgs\nGU5+LpagjDEBp0cPyJULXnnF60gMACNGaPHDadMgXz6/HdaGmRtjAsrs2TBliq6YW6yY19EYAIYO\n1XlPl13m18NaC8oYEzBOnYKuXXVYedeuXkcT5uLidNGtLVsgWzYoXtzvIVgLyhgTMEaNglWrdEmN\nvHm9jiaMOQcdO+os6Tp1oHRpT8KwFpQxJiAcOgS9e8NNN0HLll5HE+ZeflmTU9++8MADnoVhLShj\nTEB4803YtUsLw9qkXA8NG6bLZrRvD336eBqKtaCMMZ7btAkGDoS2bXUks/HIiRNavqNxY/jwQ88/\nKVgLyhjjueeegxw54I03vI4kzOXOrUVg8+XTfxCPWQvKGOOp2bNh4kR4/nkoUcLraMLUunXQuTOc\nPAlFi0L+/F5HBFiCMsZ46ORJvS6WK6eTc40Htm+Hhg31U8LOnV5Hcwbv23DGmLA1ZAj89puWebNh\n5R44eBAaNdJ1nebOhVKlvI7oDJagjDGe2LFDRzHfeSc0aeJ1NGHoxAlo3lw/IXz7LVSv7nVEZ7Eu\nPmOMJ3r00Gvk4MGeDxYLT7/8AosXw8iR0KCB19GkyFpQxhi/mzdPi2L37g1XXul1NGGqVi344w9P\nShillbWgjDF+FR+vAyNKl4aePb2OJgwNHKitJgjo5ASWoIwxfvbBB/DrrzBokF9XbjCgxQ67dYOZ\nM7XeXoCzBGWM8ZsdO7R6ToMG0KyZ19GEmalTtXzR7bfDZ58FxY0/S1DGGL/p2hWOH9fh5UFwfQwd\nc+dCq1ZaR2ryZF0NMghYgjLG+MX33+syGr16QWSk19GEmVmz4Ior4LvvICLC62jSTFwQ9EOmV1RU\nlIuOjvY6DGOMz9GjULky5Mmjo5tz5/Y6ojDhnDZVnYPDh6FgQb8eXkRWOOcyXP7XWlDGmCz3yiuw\nebOu5GDJyU+2bdPFtX79VZOUn5NTZrB5UMaYLLV6NQwYAO3awS23eB1NmNi/X0eibNmiS7cHKUtQ\nxpgsk5AAHTpA4cLwzjteRxMmYmO1ftQff8CMGVCjhtcRZZglKGNMlhk2DJYuhc8/1yRlslhcHLRo\nAcuXa3Xy227zOqILYvegjDFZYscOeOEFqFcP2rTxOpowcfSoVij/+OOQmGhmLShjTJZ4+mktBhsA\nK4eHvoQEXVzrootgwYKAWA03M1gLyhiT6SZN0h6mPn2gQgWvowlxzumngcaNtYsvRJITWIIyxmSy\nffvgqafguutsldws55z2ow4dCtWqQc6cXkeUqUIn1RpjAsIzz+go55kzQ+56GXhefx3efhueeEK/\nhlhfqrWgjDGZZvp0+OILePFFqFrV62hC3Pvvw0svwUMPaQsqxJITWIIyxmSSAwegY0eoUkXr7Zks\nFhWls59HjoRsoXkpty4+Y0ym6N4ddu7UVR2CpFh2cNqwQUee3HCDPkKYJ2lXRAqLyBQROSIif4lI\n61S2rS4i80QkVkR2iUgXf8ZqjDm/WbNgxAgdFBGV4dKg5rymToWrr4YxY7yOxC+8akENBeKAy4Bq\nwLcisso5tzbpRiJSBJgBdAUmArmAkn6O1RiTisOHdR28ihWhb1+vowlhs2bBffdp6aK77/Y6Gr/w\ne4ISkfxAC6CKcy4WWCAi3wAPAS8k2/xZ4Afn3OmPCyeA3/0WrDHmvHr0gL//1vmhefJ4HU2Imj1b\nk1KlSrqwVoECXkfkF1508UUC8c65mCTPrQIqp7DtDcB+EVkkIrtFZJqIlE5ppyLSQUSiRSR6z549\nWRC2MSa5777TenvdukHt2l5HE6L+/lsn4ZYvDz/+CBdf7HVEfuNFgooADiV77iCQ0keCksDDQBeg\nNLAJGJvSTp1zw51zUc65qKJFi2ZiuMaYlOzbB489pqP2Xn3V62hCWKlSMHgw/PQThNm1zYt7ULFA\n8pWzCgKHU9j2GDDFObccQEReAfaKSCHn3MGsDdMYcy7OwZNPapL6/nvr2ssSS5ZA9uxQs6be5AtD\nXrSgYoAcIpK0QldVYG0K264Gkq5JH3rr0xsThMaOhQkTdKXcatW8jiYELVsGDRtqzSgXvpc9vyco\n59wRYDLQT0Tyi8hNQFPg8xQ2HwU0E5FqIpITeAlYYK0nY7yzdSt06qT3nJ57zutoQtCKFboabpEi\nMGVKSFaISCuvph8/BeQFdqP3lJ50zq0VkToiEnt6I+fcz8CLwLe+ba8EzjlnyhiTtRIS4NFHdWWH\nTz/VHiiTif73P7j9dh0IMXs2lAzvWTWezINyzu0H7knh+fnoIIqkz30IfOin0IwxqfjgAx1I9tFH\ncOWVXkcTgl5/XYeQz54NpVMcsBxWrNSRMSZN1q3TLr1GjaBDB6+jCVGffQZ79kCZMl5HEhBCs8Kg\nMSZTnTgB998P+fLBJ5+E9W2RzLdmjS7PfuiQnmBLTomsBWWMOa8XXoBVq+Cbb6B4ca+jCSGrVkG9\nepA7N+zeDQWTz8AJb9aCMsak6vvv4d13oXNnaNLE62hCyMqV8J//QN68MHeu3dRLgSUoY8w57dwJ\nDz8M11wD77zjdTQhZPlybTlFRFhySoV18RljUpSQoMnp8GGYM8eqRWSqPHkgMhK++sruOaXCEpQx\nJkXvvgszZ8KHH+oSRCYTbN6sCemaa7SUkY02SZV18RljzrJypQ6MuOceXcbdZIK5c7Wy7nvv6c+W\nnM7LEpQx5gyHD8MDD8Cll9qQ8kzz8886gax0aWjVyutogoZ18RljEjmnk3D/+EOvqZdc4nVEIWDm\nTGjaVAdC/PSTZn6TJmluQYnIZSIyWEQ2isgJEdkmIt+LyJ0XGoSIlBURJyJRF7ovY0zGDRsG48bp\n+k633up1NCFg+3ZNThUravkiS07pkqYWlIiUBRaiazb1RFfAzQbUAz5CFxM0xgSxlSuhSxe44w69\n/2QyQfHiMHKkVie35mi6pbUF9YHva5Rz7ivn3Hrn3O/OufeBawFEpLSITBGRw77HZBFJLMUrIqVE\nZKqI7BeRoyKyTkTu9728yfd1ua8lNSczfjljTNocPAgtW+oH/M8/h2x2d/rCjBypLSbQG3qWnDLk\nvH+GIlIYuAMY6pyLTf66c+6AiGQDpgKXAbf5HsWBr0USb7F+AOTzvVYZeAY44Hvtet/XO4BiQPOM\n/kLGmPRxTpdu/+sv7d4rUsTriILcu+/qCf3gg/Nva1KVli6+KwEBfk9lm3poS6q8c24zgIi0Bv7w\nvfYjUAaY5Jxb5XvPpiTv3+P7us85tzPN0RtjLtiQITBpklaKuOkmr6MJYs7Ba69Bnz7QogV88YXX\nEQW9tDTk0zLI9Cpg++nkBOCc+xPYDpye4jcY6C0ii0XkNRGpkd5gjTGZa9ky6N5da+x16+Z1NEHM\nOV2LpE8faNtWm6K5c3sdVdBLS4LaADg0CWWEA3DOjQDKocu4RwKLRKRvBvdpjLlAe/bAvffqffzR\no22+0wU5dQrWr4ennoJRoyCHzeDJDOdNUL7Vb38AOotIRPLXReQitPuvuG+03+nnr0DvQ/2WZF9b\nnXPDnXP3AX2A08uexfm+2gLSxvhBfLyu77RnD0yeDIULex1RkIqPh/37NSFNnAjvv28jTDJRWs9k\nJ7SrL1pEWopIRRGpJCJPAqvRe0yrgTEiEuWbzzQGWAn8DOCbQ3WHiFwhItXQARGnk9du4BjQ0Dff\nqlCm/YbGmLP07KkTcT/6CKpX9zqaIHXihA59/M9/9PtcuawZmsnSlKB895OqA7OAt9Bk9DNwN9DB\nOeeApuhgh9m+x07gHt9rp481BE1Ks4BdwMO+/ccDTwPt0ftWUzPhdzPGpGD8eOjfHzp10mrlJgMO\nH4a77oKvv9YRe3a/KUvIv/kjdERFRbno6GivwzAm4KxZA7VqwXXXaQsqVy6vIwpCu3fDnXfCL7/o\nfKe2bb2OKGCJyArnXIYrBNmdPGPCxIED0KyZrio+YYIlpwx76CH47TeYOlVbUSbLWIIyJgwkJMCD\nD+pyRHPmQLFiXkcUxIYMgb17oXZtryMJeTbcxJgw0K8ffPstDB5sk3EzZMECnSjmnK6Ea8nJLyxB\nGRPiJk+GV17RARFPPul1NEHom2/g9ts1w//zj9fRhBVLUMaEsP/9T2+Z3HCDDim3UdDpNGoUNG+u\nS7QvWGATxvzMEpQxIWrnTl2KqHBhmDIF8uTxOqIgM2AAtGsH9erpkEerout3NkjCmBB0/LiO2Nu3\nTz/4X3651xEFoVKloE0bHUpuQx49YS0oY0LM6WXblyyBzz7TOU8mjY4f12GOAPfdpxXJLTl5xhKU\nMSHm7bd10cF+/XTVB5NG+/ZB/fq6pPC2bV5HY7AuPmNCyjffaJ29Vq2gd2+vowkimzZBo0Y6Uezz\nz6FECa8jMliCMiZkrF6tt0xq1NDBZzZiL42io7UixMmT8OOPcPPNXkdkfCxBGRMCtm3Ta2zBglq/\nNG9eryMKImPGQL588P33UKmS19GYJOwelDFB7nRh7QMHdC6p9U6l0YED+vWdd3RpYUtOAccSVCpE\nJNXHI4884nWIzJgxAxEhNjbW61CMB06e1MFma9ZoAdhq1byOKAg4B716QdWqWpk8Rw4oWtTrqEwK\nrIsvFTt27Ej8fvr06Tz++ONnPJc3g/0oJ0+eJGfOnBccnwlvzumaTjNmwPDhOvjMnMexYzr5dtw4\nHYtvlSECmrWgUnH55ZcnPi666KKznitUSBf+ffbZZ6lQoQJ58+alXLly9OrVi7i4uMT9vPDCC0RF\nRTF8+HDKlStH7ty5iY+P59ChQ7Ru3Zr8+fNTrFgxBgwYQP369XniiScS33v8+HG6detGiRIlyJ8/\nP7Vq1eLnn38GYN26dTRq1AiAAgUKICJnvNeEtrfego8/1lF7jz/udTRBYNcuXf12/Hj4v//T2k85\n7DN6IPPkX0dECgMjgAbAXqCnc+7LVLbPBawCCjjnSvonyrQrVKgQn332GcWKFePXX3+lY8eO5MuX\nj169eiVus27dOqZOncqUKVPIli0b2bNn5+mnn2bJkiVMmzaNSy+9lD59+rB8+XKuvPLKxPe1adOG\n3bt3M378eIoVK8bUqVNp1KgRv/zyC5GRkXz55Ze0bt2ajRs3ki9fPvLly+fFKTB+Nm6cJqb774fX\nXvM6miDRqROsWgWTJmmZDRP4nHN+fwBjgfFABHAzcBConMr2vYB5wNa07L9GjRous02YMMHp6Tq/\nQYMGucqVKyf+/Pzzz7vcuXO7ffv2JT63b98+lz17djdlypTE5w4cOOAiIiJcx44dnXPOrV271mXL\nls3t3LnzjP03bNjQde3a1Tnn3Pfff+8Ad/jw4Qz/bia4zJvnXK5cztWp49zx415HEwQSEvTr9u3O\nrVjhbSxhBoh2F5Ar/N6CEpH8QAuginMuFlggIt8ADwEvpLB9OeBB4FngY3/GmlZjx45lyJAh/Pnn\nn8TGxhIfH0+uZOVRypUrR+Ek/d0bNmzg1KlTXH/99YnPFSpUiEpJRhKtWLGChIQEypcvf8a+Tpw4\nQe7cubPotzGBbO1aLQBbtqwWgLU/g/MYMgRmztSTVayYrdQYZLzo4osE4p1zMUmeWwXceo7thwAv\nAsdS26mIdAA6AJQuXToTwkybOXPm8NBDD/Haa69Rv359ChUqxIQJE+jXr98Z2+XPnz/d+05ISCBn\nzpz873//Q5LNuszI/kxw27IFGjbUpDRjBlxyidcRBbD4eHjmGRg6FO6+G+Li7H5TEPLiXywCOJTs\nuYNAgeQbikgzILtzboqI1E1tp8654cBwgKioKJc5oZ7fwoULKV++PC+88G/jb/Pmzed9X4UKFcie\nPTvLly+nadOmABw6dIh169ZRo0YNAKpXr87JkyfZu3cvN954Y4r7Od1SO3Xq1AX+JiaQ7d0LDRpA\nbCzMmwflynkdUQA7eFBrPf3wA3TvrgMismf3OiqTAV4kqFigYLLnCgKHkz7h6wp8G7jTT3FlSGRk\nJJs2beKrr76iRo0afPvtt0yaNOm87ytcuDAPPvgg3bp1o1ChQhQtWpSXX36ZbNmyJbaWrrnmGlq0\naEGbNm0YMGAA1apVY+/evfz8889cffXVNGnShLJlywI6DL5Bgwbky5fPWlchJjZWJ+Ju3qy9Vdde\n63VEAcw5aNIEFi/WIY7t23sdkbkAXgwzjwFyiEiFJM9VBdYm264CUBaYLyI7gclAMRHZKSJl/RBn\nmtx7773897//5amnnqJatWosWLCAl19+OU3vfe+996hZsyZ33nkn9evXp3bt2lSpUoU8SVaWGzNm\nDK1bt+bZZ5+lYsWK3H333SxZsiSxG/OKK66gV69ePPvss1x22WV069YtS35P4424OLj3Xi0XN348\n3HKL1xEFOBFd3/6HHyw5hQDRgRZ+PqjIOMAB7YFqwHdAbefc2iTb5ACSLmFZG3gfqA7scc6ds08r\nKirKRUdHZ0XoWerYsWOULFmSfv360alTJ6/DMR5LSNDl2r/80hoDqXIOBg/WtZxeOGuclfGQiKxw\nzkVl9P1e3TV8ChgJ7Ab2AU8659aKSB3ge+dchHMuHth5+g0ish9IcM7tTHGPQWjZsmVs2rSJqKgo\nDh48yOuvv87Jkye59957vQ7NeMw56NZNk9Prr1tyOqfjx+GJJ+DTT3Xxq4QEyGb1B0KFJwnKObcf\nuCeF5+cEZxsOAAAgAElEQVSjgyhSes8cIOAm6V4I5xxvvfUWMTEx5MqVi+uuu4758+dz2WWXeR2a\n8dhrr8G778LTT+uEXJOCHTt0wu3SpdC3L7z0kiWnEGPjLj1Uq1YtVq5c6XUYJsAMGgR9+kDbtvq9\nreuUgsOH4frr4Z9/tDJE8+ZeR2SygCUoYwLI8OHw7LM6MGLECGsQnFOBAtq0rFMHrrnG62hMFrE/\n/yD0yCOP0LhxY6/DMJlszBi9nXLnnfq9zStN5uRJzd6zZ+vPTz1lySnE2X+BIDR48GCSjr6sW7cu\nVapU4f333/cwKnMhpkyBhx+GW2+FiRMhWaUss2OHLny1YIG2nm67zeuIjB9YggowcXFxZ9XxS+70\nMh8mNPzwgxY+qFkTvvnGlms/y/z5mpwOHdJhjQ884HVExk+si+885s2bxw033EBERASFChXi+uuv\nZ82aNYwePZqIiAimTZtGZGQkefLk4bbbbuPPP/9MfO/GjRtp2rQpl19+Ofnz56d69epMnz79jP2X\nLVuWvn370q5dOy666CLatGkDQL9+/ShTpgy5c+fm8ssvp23btonvSdrF98gjjzB37lyGDh2auNLv\npk2buPLKK+nfv/8Zx9qwYQMiYgMzAsi8eToQrXJl+O47bRyYJBYu1DWcIiJgyRJLTmHGElQq4uPj\nadq0KTfffDOrVq1i6dKlPPPMM2T31fU6ceIEr7zyCqNGjWLx4sWcOnWK5s2bJ3a/xcbG0qhRI2bN\nmsWqVato0aIFzZs3Z926dWccZ+DAgVSqVIno6GjeeOMNJk2aRP/+/fnggw/YsGED06dPP6PqeVKD\nBw/mxhtv5NFHH2XHjh3s2LGD0qVL89hjjzFq1Kgzth05ciTVqlWjevXqWXC2THrNn6/3m8qU0RJG\nF1/sdUQB6IYbdPh4dLTdbwpHF7JWR6A+Mms9qH379jnAzZkz56zXRo0a5QC3YMGCxOc2b97ssmXL\n5mbNmnXOfdaqVcu9+uqriT+XKVPGNW7c+IxtBgwY4CIjI11cXFyK+3j44YfdXXfdlfjzrbfe6jp1\n6nTGNjt27HA5cuRwixcvds45Fx8f74oXL+6GDBmSym9s/GX+fOfy53euYkVdpsgkERPj3O2324kJ\nAVzgelDWgkpF4cKFeeSRR2jYsCF33XUXAwcOZMuWLYmvZ8uW7YyWTZkyZShevDi//fYbAEeOHOG5\n557j6quv5uKLLyYiIoLo6Ogz9gEQFXVmJZCWLVty/PhxypUrx2OPPcaECRM4ceJEumK//PLLady4\nMSNHjgRgxowZ7N+/P7EL0Xhn4UJo1AhKltQBabZEURJTp0JUFKxcqdVxTVizBHUeo0aNYunSpdxy\nyy188803VKxYkR9++CHx9eTrNCXVvXt3JkyYwKuvvsrcuXP55ZdfuP7664mLiztju+TVx0uVKsX6\n9esZNmwYBQsWpFu3btSoUYMjR46kK/b27dszfvx4jh49ysiRI2nWrBkXWz+SpxYtgjvugOLFLTmd\n4eRJ6NED7rkHIiNhxQo4xxIzJnxYgkqDqlWr8vzzzzNnzhzq1q3Lp59+CuiCgsuWLUvcbsuWLWzf\nvp2rrroKgAULFtC2bVtatGjBtddeS8mSJdm4cWOajpknTx7uuusuBg0axPLly1m7di0LFy5Mcdtc\nuXKluB7UHXfcQcGCBfnoo4+YNm0a7dq1S++vbjLR4sWWnM7pueegf3/o1ElvzpUp43VEJgDYMPNU\nbNq0iWHDhnH33XdTokQJ/vzzT1avXs2TTz4JQI4cOXjmmWcYPHgwefPmpWvXrlSuXJn69esDulbU\nlClTaNq0KTlz5uSVV17h+PHj5z3u6NGjiY+Pp1atWkRERDB+/Hhy5sxJhQoVUty+bNmyLFu2jM2b\nNxMREUHhwoXJli0b2bNnp127dvTs2ZMSJUpQr169zDs5Jl0WL9bVcC+/XJNT8eJeRxQgTp6EnDnh\n+efh5pu14KsxPtaCSkW+fPmIiYmhZcuWREZG8vDDD9OmTRuef/55AHLnzk2vXr1o27YttWrVIiEh\ngcmTJyd2+w0cOJBLL72UOnXq0KhRI2644Qbq1Klz3uNedNFFjBgxgjp16lClShUmTZrE5MmTKXeO\nZVS7d+9Orly5uPrqqylatOgZ97jatWtHXFwcjz76aKrdkSbrLFhgyeksp7v0GjWCU6f05FhyMsl4\nsh5UVvPHelCjR4+mc+fOxMbGZulxLtTSpUu56aab+PPPPxMXOTT+8+OP0LQplCoFP/0EJUp4HVEA\n2LJFZyYvWaLligYNstIZISpY14MyWezEiRP8/fff9O7dm2bNmlly8sD06Vr0NTISZs0CW0UFPSlt\n20J8vC4RfN99XkdkAph18YWosWPHUrFiRfbt28fAgQO9DifsTJigFSKuuQbmzLHkBMCRI/D441C2\nrA4jt+RkziMku/iKF49yy5ZFUzKkljc0weKzz+DRR3WU9LffQtiXTty4UUfl5cgBa9dC+fKQJ4/X\nURk/uNAuvpBsQe3YAZs2eR0F9O/f/6x6eCa0ffSRViX/z3+0CGxYJyfn4OOPtRk5YIA+V7myJSeT\nZiF7Dyqdc1qzRPfu3b0OwfjRwIHQrRs0bqxdfGF9Hd63T7vzpkyB22/X+07GpFNItqAgMBLU+vXr\nWb9+vddhmCzmnNYz7dYNWrbUFcjDOjnNnw9Vq+qAiP79YcYMm5VsMiRkW1CBMPq7Y8eOAMyZM8fb\nQEyWOXVKR0oPHw6PPaZdfGG/Em5cnK4b8s03YJXzzQWwFpQxGXT8uA5EGz4cXnxRb7eEbXKKiYFP\nPtHv69WDX3+15GQuWMj+dwqEFpS1nELXwYNa13TOHHj3XejSxeuIPJKQAB98oLX08ufXiV8XXRTG\nmdpkJmtBGZNOO3dC3bpawmjMmDBOTn//DQ0awH//qydk1SpNTsZkkpD8mJMtW2AkqA4dOgAwfPhw\njyMxmWXjRr0m79wJ06ZpdfKwdOAAVKsGJ07AsGE6Ys9qPZpMFrIJKhC6+GJiYrwOwWSilSt1ifaT\nJ+Hnn6FWLa8j8sDRo5Avn7aU3nkHbr1VJ94akwVCNkEFQguqW7duXodgMsn06XD//XDJJVqR3Lfk\nV3j55hvo2BE+/xzq1wdbX8xksZBNUIHQgmrSpInXIZhM8OGH0Lmz9mhNnx6GU3r274euXbWGU7Vq\nujSGMX4QkoMksmcPjBbUtGnTmDZtmtdhmAxKSNAli556Srv25s4Nw+T09ddw9dU6GqR3b1i6FKpU\n8ToqEyasBZWFBvjqj1lLKvgcO6bVeSZO1FXIBw/WDz5h57ffNCt//z1cd53X0ZgwE7IJKhBaUJGR\nkV6HYDJgzx5dZHDJEq1x2rVrGA1Qcw7GjtUqt3fdpfObevTQZdmN8bOQTFCB0sVnw8uDT0yMXpe3\nbtWCr2G1Cvm2bfDEE3qjrVkzPRE24dZ4KCTvQQVKF58JLjNn6tDxAwd0GHnYJCfntEzR1VfruvSD\nBml2NsZjIZmgsmfXUjReq1u3LnXr1vU6DHMezuk9pkaNoFQpWL5cFxsMG1Om6ETb6tW1ht4zz4Tp\nDTcTaEIyQeXIoYU8jx71OhIT6OLioEMHvSY3aQKLFumK5CHv2DGIjtbv77lHR4P89JNNujUBJSQ7\nmE93m+/bp5PevTJs2DDvDm7Oa88e7cabPx969YJ+/bR7OOTNmgVPPqnzm/76S5fGCJv+TBNMQvK/\n4+kEtXevt3FUrFiRihUrehuESdHq1VCzpnbnffklvPZaGCSnnTuhdWstJpgtm95nKlDA66iMOSdP\n/kuKSGERmSIiR0TkLxFpfY7teojIGhE5LCKbRKRHWvYfKAmqf//+9O/f39sgzFmmTIHatbWm3rx5\n8MADXkfkB5s2aX2mSZPg5Zc1Q9er53VUxqTKq8+MQ4E44DKgDfChiFROYTsB2gIXA3cAnUXk/vPt\nPFAS1PTp05k+fbq3QZhEp05Bz57QvDlUrqytp5o1vY4qi+3Zo1/LltUZx6tXQ9++Yb4mvQkWfr8H\nJSL5gRZAFedcLLBARL4BHgJeSLqtc+7tJD+uF5GpwE3AuNSOkfQelJcaN27sbQAm0d692lL68Ucd\nFPHee5A7t9dRZaE9e3SZ37FjtRpE6dLaj2lMEPFikEQkEO+cS7oWxSrg1tTeJCIC1AFSHHkgIh2A\nDgClS5dBxPsWVPfu3b0NwAA6WK1FC9i1C0aMCPEi3PHx8NFH8NJLOhnw6adtEUETtLxIUBHAoWTP\nHQTOd7e2L9olOSqlF51zw4HhAFFRUS421vsEtX79egAbKOGhESO0Z+uyy3QF3KgoryPKQrGxcNNN\n2o1Xv742E8NyXRATKrxIULFAwWTPFQQOn+sNItIZvRdVxzl3Ii0HKVpUPzF7qWPHjgDMmTPH20DC\n0IkTuhL5xx/D7bfrSL0iRbyOKovExkJEhD7q14c+ffRGW9gUEDShyotBEjFADhGpkOS5qsDalDYW\nkXboval6zrmtaT1IiRJaWsyEn40btSHx8cd6G+b770M0OR05opO3SpaEtb7/PgMGaH+mJScTAvze\ngnLOHRGRyUA/EWkPVAOaArWTbysibYA3gNucc3+m5zilSunEeC9Zy8n/JkyA9u11ms/XX2tV8pBz\n6pQuHti7N2zfrgkpIsLrqIzJdF4NM38KyAvsBsYCTzrn1opIHRFJWub1NeASYLmIxPoeH6XlAKVK\nwY4des/YhL7jx3Vhwfvu05qnv/wSoskpLk4r2rZrp3/kCxZomaIyZbyOzJhM50mpI+fcfuCeFJ6f\njw6iOP1zuYweo2RJ/aC5c6d+74UOHToAtuxGVouJ0cS0ahV07w5vvBGCyxdt3w7Fi0OuXFo0sHt3\naNXKuvJMSAvZ4i6lSunXv//2LoaYmBhiYmLOv6HJsC+/hBo1dP2m6dPhnXdCLDnt3q1NwzJlYPFi\nfe7ll+H++y05mZAXksVi4d9W099/e7d0Qrdu3bw5cBiIjYUuXWDkSB0QMW6cdy3lLHHokA54GDhQ\nK48/8QRceaXXURnjVyGboMr5Ogc3bvQuhiZNmnh38BC2dCm0aQN//qmli/r1C7GFX+PioEoV/XTV\nooVWgKhUyeuojPG7kO3iK1AAihXT+xNemTZtGtOmTfMugBATH6/J6KabtNDrnDl6vykkklN8vA47\ndE7vM73yihYLnDjRkpMJW6HwX/ucKlSADRu8O/6AAQMAa0llhj//hAcf1NswrVvD0KEhUsEnIUHH\nxr/0kv6xzpsHderAo496HZkxngvZFhRAZKS3LajIyEgiIyO9CyAEOAejR0PVqjoXdcwYfQR9ckpI\n0HU/qlfXAQ+5cmkL6uabvY7MmIAR0i2oyEgt6vzPP3Dxxf4/vg0vvzB79ugAtokT4ZZbdG5qyEz3\nOXxY5zIVLaq/WOvWkD2711EZE1BCugVVpYp+/fVXb+Mw6Tdhgk64nToV3nwTfv45yJNTQoIuFnj/\n/fp9oULanffbb/DQQ5acjElBSCeoqlX166pV3hy/bt261K1b15uDB6k9e3TS7X33aUJauRJeeCGI\nr98JCdoErFoV7r1XS1xs366vXXNNiIzwMCZrhHSCKlZMi4R6laBM+pxuNX39tY6sXrz431ZwUFq3\nTpfubdlSR+mNGaM30kJqwpYxWSekP76JQLVq+incC8OGpbi2oklmzx5ds2nCBK0KMXp0ECem2Fgd\ncnjttdoELFlSKz+0bBnEzUBjvBHSCQqgdm39NH7oEBRMvgpVFrOFClPnnDYqunaFgwf13+m554K0\nVNHevfD++zBkiA4xjImBvHlh1iyvIzMmaIV0Fx/olJKEBFi0yP/H7t+/P/379/f/gYPAH39AgwY6\nPqB8eVixAnr1CsLk9Ndf8Mwz2lp65RX9g/viC2stGZMJQj5B3XCDXivmz/f/sadPn8706dP9f+AA\nFhcHr7+uXXjLlumE24ULdbxA0HDu33Vc5szRX6JlS72/9PXX3hV/NCbEhHwXX0SE3tfwIkE1btzY\n/wcNYAsXQseOeh2/914YPFhXkAgacXF6o+zdd3W4eLdu+rVePRv4YEwWCPkEBdrrMmSILmqXJ4//\njtu9e3f/HSyA7d+vRV2HD4fSpWHaNAiq3L1vHwwbpi2l7duhYkUoUUJfy53bkpMxWSTku/gAbr1V\nP/wuXOjf465fv57169f796AB5NQp+OgjrYn4ySc6GGLt2iBLTqAVxXv10n7J777TybX33+91VMaE\nvLBoQdWrpwOqJk/W7/2lY8eOAMyZM8d/Bw0QCxfCf/8L//uffkAYMiRI7jMdOQJjx+pCU9OmwSWX\naCmLggV1TpMxxm/CogWVLx80aqS1ORMSvI4mtO3YoSPzbr5ZF4MdNw5mzw6C5PTbb5pRixeHxx/X\n+Uynl2O+8UZLTsZ4ICxaUKC9NJMnw5IlOjfKH8Kp5XTsmA56eP117U7t2RNefFEHqQS89es1AeXK\npaPxnnxS/0hsSXVjPBU2Cequu3SOzYQJ/ktQ4SAhQSfb9uqlDY7GjWHQoABendw5nRQ3apSOmHn/\nfR30MHKkBl+0qNcRGmN8wqKLD7R49N1368oGx47555gdOnSgQ4cO/jmYB2bPhpo1oW1bva7//LPe\ntgnI5LR9O/zf/+nqtDffrH2PST36qCUnYwJM2CQo0LWF9u/XVpQ/xMTEEOPliolZ5LffoEkT+M9/\ntMLPF1/o6uS33eZ1ZMnExmqLCaBvX+13vPRSbS3t3KmtJ2NMwBJ3+j9wCImKinLR0dFnPe8cXHWV\nlkpbsiTr45g2bRoQOku+b9yo1XzGjIECBfQe09NP+3du2XnFxcGMGfDll/DNN9rMq1ULNm3S6g8V\nKngdoTFhQ0RWOOeiMvr+sLkHBXrPu1MnvajOnavDn7NSqCSmLVu0kOvIkTqO4Nln4fnndSmTgLFj\nh1YNnzhRl1AuUkS77QoX1tfLlfM2PmNMuoVVCwr0/lP58nqfZO7crB2oFewtqB074I03tAIEaJmi\nnj11nS3PnTyp/4DOwe23a7n68uXhjjt0+fT69YOw8qwxocVaUOmUN6+OOOvcGWbOhIYNs+5YAwYM\nAIIvQW3ZAv37w8cfa69Yu3bQuzeUKuVxYMeP6/IVkydr993+/VC3riaoggU1o9oKtcaEjLD839y+\nPQwcCF266Gq7uXNnzXEiIyOzZsdZZN06eOstHfQAOuG2d2+44goPg4qN/XcyVaNGWj389JDMFi10\nzY7TLDkZE1LCrovvtBkz9HrXt6/eughnK1ZoNZ/Jk3XAw+OPa6Hu0qU9CMY5WL0avv1W696tXKkl\nKSIi4IcfdJvbbtObYcaYgGZdfBl0xx3wwANa+aBRI7j+eq8j8q+EBL3+v/su/PSTNkpefFFblZ5N\nB/r6ay03tHWr/lyjBvTooSPzIGv7Y40xASes5kElN3Soll5r1UoHfmW2unXrUrdu3czf8QU4dEhL\nEkVG6lymdet0/urpkXp+SU5xcTBvHvTpAzfd9O+y6MWL6yeFESN0Ym10tI5rPz0SzxgTVsK2BQVw\n8cUwfryuF9WsmfYgZdX9KK+tWwcffKAVfmJjtdzT669D8+Z+HOy2bZveAJw3D44ehWzZtBTF6dVp\nr78eJk3yUzDGmEAX1gkKdA7n6NHQpo2OTh47NvNubwwbNixzdpRBR45o1YxPPtHlL3Lm1GWMnn4a\nojLcK5wGR4/qeu4LFuijdm1tLRUpArt2wWOP6bont96qs6aNMSYFYZ+gQBPTnj3wzDPaohg/HvLn\nv/D9VqxY8cJ3kk7Oac/YiBFaTOHwYa2F+s47Oirvssuy4ICHDulNLOc08cyfr60iEV3k7/TJzJ1b\nBz0YY0waWILy6dJFR7A9+aQu/zN58oUXPe3fvz/gn6Xf163T1t+4cRATo/O97rtPe9RuuikTJyTv\n2qWF95Yv10y4fLkO94uO1oNUraon8Kab9OvFF2fSgY0x4SZsh5mfyw8/aIvq1CkdTNC2bcYv7qcH\nSGTVulDr1+vAt3Hj4JdfNM66dXV04n33aaMmw06e1Ez36686qu50km3USMfoZ8umhQ1r1tREFMJV\n240xGXOhw8wtQaVg82a9J7VokV7wBw2CatXSv5/MbkHFx+stnWnT9LFhgz5fq5YmpZYtdSBcupw4\noYVUK1bUDDd0qNY2+v13TVKgN+UOHtQm5qJFOka9WrUgWY3QGOMVS1ApiKpUyUXPnKlX6wxWF0hI\n0MEFzz8PBw5o4YJu3XTEn78WWk1IgDVrtCD37Nlaeu7AAc0Xt92mw8QbN4YyZc6zo9hY7fPLnl0r\nMUycqK2jDRt0fHlCgnbdXXopfPghTJ+ua7Rfc43eQ6pUKXSHNxpjskxQJigRKQyMABoAe4Gezrkv\nU9hOgP8D2vue+gR4wZ0n6CgRFw3aDVWihF7B+/XTq/quXbB4sV6ML7tMH6m0BA4cgCFDtBX1zz96\nX6p1a12hNypKD3Eu69evB9I+WGLXrn9v60RHw9Klut4SaLmh226DO+/U0nMFCqCJZd8+rbRwxRWa\nhBYt0tERW7bo4++/tWbdunXaSnrvPXjpJZ0IVaHCv4+mTX07NcaYzBGslSSGAnHAZUA14FsRWeWc\nW5tsuw7APUBVwAGzgE3AR6nuvUIFeO45+Ouvfy/Upyf7LFmik56SypdP59/ccYdmhvfe0+HPhQpx\nUaFCvFTsIrotbcTExSWYMPwfpr36F+P75SFXgTxUqpqbyjXyUL5aAUqVy0Hp0nDJJTpwrWPHjgDM\nnj2H+HhtyOze5diz/SR7t53g7z9OsCXmOH//cYKlW4qxeXc+irGdKFZQpcRBWpU/SOVbD1L+0kMU\neu4JKFtW+/Zu7q3ZbM8eTVKg2ez667VVNGaMDlwoXVqHeJcu/e8Nqaee0moN/moGGmNMBvm9BSUi\n+YF/gCrOuRjfc58D25xzLyTbdhEw2jk33PfzY8DjzrkbUjtGqvegDh/Wi/iuXf8+du/Wm/wVK2r9\nn6ee0nsuBw/+uyLrjz/qEOrx43UyUTL1mcVP1KclX/ElrROfd2giaMBM5nAbD/AlX9LmrPe/3ehn\nste7jTsPjuWqV1uf+WLOnFpt4dZbtZ9v0KB/W4CXXqqPevV0npFzlnyMMQEh6Lr4ROQ6YKFzLl+S\n57oDtzrnmiTb9iDQwDm31PdzFDDbOXdWX5SIdEBbXABVgDVZ9CsEuyJot6o5m52bc7Nzc252bs6t\nYkrX67TyoosvAjiU7LmDQEq/RITvtaTbRYiIJL8P5WtlnW5pRV9I1g5ldm7Ozc7Nudm5OTc7N+cm\nIhkfTo03xWJjgYLJnisIHE7DtgWB2PMNkjDGGBP8vEhQMUAOEamQ5LmqQPIBEvieq5qG7YwxxoQY\nvyco59wRYDLQT0Tyi8hNQFPg8xQ2/wx4VkRKiEhxoBswOg2HGZ5Z8YYgOzfnZufm3OzcnJudm3O7\noHPj5TyokcDtwD50btOXIlIH+N45F+HbToC3OHMe1PPWxWeMMaEvJCtJGGOMCX5hvaKuMcaYwGUJ\nyhhjTEAKygQlIoVFZIqIHBGRv0Sk9Tm2ExF5S0T2+R5v+e5rhax0nJseIrJGRA6LyCYR6eHvWP0t\nrecmyfa5ROR3Ednqrxi9kp5zIyLVRWSeiMSKyC4R6eLPWP0tHf+ncovIR75zsl9EpolICX/H608i\n0llEokXkhIiMPs+2XUVkp4gcEpGRInLeCtRBmaA4s5ZfG+BDEamcwnZJa/ldCzQBOvorSI+k9dwI\n0Ba4GLgD6CwiZ9dwCi1pPTen9QD2+COwAJCmcyMiRYAZwDDgEuBKYKYf4/RCWv9uugA3otea4mhJ\ntyH+CtIj24HX0EFv5yQiDYEXgHpAGeAK4JXz7t05F1QPID/6xxKZ5LnPgf9LYdtFQIckPz8GLPH6\ndwiEc5PCe98Dhnj9OwTKuQHKAb8DjYCtXscfKOcGeAP43OuYA/TcfAi8neTnu4D1Xv8OfjpPr6F1\nU8/1+pfAG0l+rgfsPN9+g7EFFQnEO1+hWZ9VQEqfaCr7XjvfdqEiPecmka/bsw6hPQk6vedmCPAi\ncCyrAwsA6Tk3NwD7RWSRiOz2dWOV9kuU3kjPuRkB3CQixUUkH9ra+t4PMQaDlK7Fl4nIJam9KRgT\nVKbU8sui2LyWnnOTVF/0b2FUFsQUKNJ8bkSkGZDdOTfFH4EFgPT83ZQEHka7s0qjy9+MzdLovJWe\nc7MB+BvY5nvPVUC/LI0ueKR0LYbzXJuCMUFZLb9zS8+5AfQmJ3ov6i7n3IksjM1raTo3vuVg3gae\n9lNcgSA9fzfHgCnOueXOuePofYTaIlIoi2P0SnrOzVAgN3pvLj9aMcdaUCqlazGkcm2C4ExQVsvv\n3NJzbhCRdvhuXDrnQn2kWlrPTQWgLDBfRHaiF5livtFHZf0QpxfS83ezGl089LRQ/bB3WnrOTTX0\nPsx+34e9IcD1voEl4S6la/Eu59y+VN/l9c21DN6QG4d2K+QHbkKbi5VT2O4J9EZ3CXRUzVrgCa/j\nD5Bz0wbYCVzldcyBdG7QJWguT/Jojo5Uuhzt9vP89/D47+Y/6Oi0akBOYBAw3+v4A+TcjAImAYV8\n5+ZFdCFWz3+HLDw3OYA8wJvo4JE8QI4UtrvDd725GrgI+Jm0DN7y+hfM4EkpDHwNHAG2AK19z9dB\nu/BObydod81+3+NtfOWdQvWRjnOzCTiJNr1PPz7yOv5AODfJ3lOXEB/Fl95zAzyJ3mf5B5gGlPI6\n/kA4N2jX3hhgN3AAWABc73X8WXxu+qKt6KSPvuj9yVigdJJtnwV2offnRgG5z7d/q8VnjDEmIAXj\nPShjjDFhwBKUMcaYgGQJyhhjTECyBGWMMSYgWYIyxhgTkCxBGWOMCUiWoIzxExGpKyIuGCsLiMgc\nEXnf6zhMeLEEZUKOb0G9UyKyMAPv7Ssia7IiriDXHOh5+gcR2Swi3T2Mx4QBS1AmFLUHPgCqiMhV\nXgrHpg0AAAUdSURBVAcT6EQk1/m2cVpfLtXCnsZkNktQJqSISF6gNTAcmIguUpl8m+IiMkZE9onI\nURH5RURuE5FHgJeByr6uOOd7Dt/39ybbzxmtCBF5VkRW+5YG3yYin4jIRemMv7lvH8d8y4bPFZHL\nfK/1FZE1ItJeRLb4tvk6aZehiNQUkZkiste3tPYCEbkx2TGciHQSkckicgR4Q0Ryish7IrLdt3z3\n3yLyf0nek9jFJyJz0FVR30lynvL7jpf8HN0uIidP/w7GpIclKBNq7gX+cs79ihavbCsiOU+/6FtO\nYy5asfwe4Br+XbNnPDAAWA8U8z3Gp+PYCcAz6OJsrYHrSceS3yJyOVqY9FN0LaFbfL9DUmWBB4Gm\nQH20+nrS5bYL+N5Tx3f8X4DvUlgY7mXgO/T3H4ouL9IMuN+3z1boeUhJc2Aret6KAcWcc0fQgqrt\nkm3bDpjunNuV6i9vTApyeB2AMZnsMf69qM8FjqIX84m+51qjlclvdM7t9T238fSbRSQWXUF1Z3oP\n7Jx7N8mPm0XkOWCqiDzsnEtIwy6Ko1WwJzrn/vI9l/x+WF6grXNuiy/ejujSIBWccxuccz8n3VhE\n/gu0QJeu/yLJS+Odc58k2a4MurTEfKcFOrcAi87xe+4XkVPA4WTn6WNgiYiUcM5tE5GL0Q8BLdPw\nuxtzFmtBmZAhIlcCNwNfAvgutGM4s5vvOmB1kuSUmcf/j4jMEpGtInIYXUsqF5oQ02IV8COwRkQm\niciTIlI02TbbTicnn6Voy+0qXwyXisgwEYkRkYPognCXotWlk4pO9vNodAmNGBEZKiJ3iUi6rg/O\nuWjgV3TFXdAPA/uxRftMBlmCMqGkPZAd2CIi8SISjy7I2EBESl3gvh26fEtSSbsOywDfouuPtQRq\n8G9313kHIQA4504BDXyP1Whi3SAiVVN945k+BWoCXYHaaNLZmkIMR5IdeyXafdgTvS58CsxKb5IC\nPgEe8X3fDvjU93sZk26WoExIEJEc6Cf3nuhF+fSjKnqxf9S36f+Aa1OZixSHJrnk9qD3W04f77Kk\nPwNRaBLo6pxb7JyLQbvs0sWpxc65V9BEsx29H3RaiWTJ9nr0//Hvvp9vBoY45751zq1FW1BJ40zt\n2IedcxOdc08Cd6GLE155js3PdZ7GACVFpDNQHV33x5gMsQRlQsVdQBHgY+fcmqQPdODBoyIiaPff\nbvTeUB0RuUJE7haR23z72QyU8c2lKiIiuX3P/wx0EpEoEbkO7RI7nuT4G9D/T8+ISDkReQAdMJFm\nInKDiPT2jcQrDdwNlAJ+S7LZMeBTEanmG533EfCtc26D7/UY4EERuVpEavp+97g0HPtZEXlARK7y\ndZW2RheW23qOt2wG6ohIiaTJ3jl3AJiADjaZlyQuY9LNEpQJFY8Bs51z+1J4bQLafXW7b7TZreiF\ndxo6COEVtAsPdMnu74Cf0FbTA77nuwF/AnPQARefoIkOAOfcaqALumrob2h3Y3onsh5ElxSfjia8\nAcCrzrmkgxs2o0lnGpo0/+Tf1iFot1oEsMK33Ujfe87nMNADWAasRFufjZxzR8+xfR80eW5Ez1NS\nI9DW5Ig0HNeYc7IVdY0JEiLSF7jXOVfF61hSIyKtgGFA8VQSnDHnZcPMjTGZQkTyoS3VF9GuVktO\n5oJYF58xJrM8hw6V3w+86nEsJgRYF58xxpiAZC0oY4wxAckSlDHGmIBkCcoYY0xAsgRljDEmIFmC\nMsYYE5D+H3X4cGCzjDwQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7b00015f8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = 0.1\n",
    "q = np.linspace(0, 1, 500)\n",
    "kl_div = p * np.log(p / q) + (1 - p) * np.log((1 - p) / (1 - q))\n",
    "mse = (p - q)**2\n",
    "plt.plot([p, p], [0, 0.3], \"k:\")\n",
    "plt.text(0.05, 0.32, \"Target\\nsparsity\", fontsize=14)\n",
    "plt.plot(q, kl_div, \"b-\", label=\"KL divergence\")\n",
    "plt.plot(q, mse, \"r--\", label=\"MSE\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.xlabel(\"Actual sparsity\")\n",
    "plt.ylabel(\"Cost\", rotation=0)\n",
    "plt.axis([0, 1, 0, 0.95])\n",
    "save_fig(\"sparsity_loss_plot\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# KL divergence\n",
    "def kl_divergence(p, q):\n",
    "    return p * tf.log(p / q) + (1 - p) * tf.log((1 - p) / (1 - q))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden = 1000  # sparse coding\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 0.01\n",
    "sparsity_target = 0.1\n",
    "sparsity_weight = 0.2\n",
    "\n",
    "activation = tf.nn.sigmoid\n",
    "initializer = tf.contrib.layers.variance_scaling_initializer()\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "W1 = tf.Variable(initializer([n_inputs, n_hidden]), dtype=tf.float32, name=\"W1\")\n",
    "W2 = tf.Variable(initializer([n_hidden, n_outputs]), dtype=tf.float32, name=\"W2\")\n",
    "b1 = tf.Variable(tf.zeros([n_hidden,]), name=\"b1\")\n",
    "b2 = tf.Variable(tf.zeros([n_outputs,]), name=\"b2\")\n",
    "\n",
    "h1 = activation(tf.nn.xw_plus_b(X, W1, b1))\n",
    "outputs = tf.nn.xw_plus_b(h1, W2, b2)\n",
    "\n",
    "mse_loss = tf.reduce_mean(tf.square(outputs - X))\n",
    "sparsity_loss = tf.reduce_mean(kl_divergence(sparsity_target, tf.reduce_mean(h1, axis=0)))\n",
    "loss = mse_loss + sparsity_weight * sparsity_loss\n",
    "\n",
    "train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train MSE: 0.089448 \tSparsity loss: 0.31594 \tTotal loss: 0.152636\n",
      "1 Train MSE: 0.0798196 \tSparsity loss: 0.301433 \tTotal loss: 0.140106\n",
      "2 Train MSE: 0.0702621 \tSparsity loss: 0.307248 \tTotal loss: 0.131712\n",
      "3 Train MSE: 0.072368 \tSparsity loss: 0.263305 \tTotal loss: 0.125029\n",
      "4 Train MSE: 0.0658286 \tSparsity loss: 0.231783 \tTotal loss: 0.112185\n",
      "5 Train MSE: 0.0573753 \tSparsity loss: 0.152271 \tTotal loss: 0.0878295\n",
      "6 Train MSE: 0.0461967 \tSparsity loss: 0.091913 \tTotal loss: 0.0645794\n",
      "7 Train MSE: 0.0323527 \tSparsity loss: 0.0477101 \tTotal loss: 0.0418947\n",
      "8 Train MSE: 0.0224417 \tSparsity loss: 0.0230307 \tTotal loss: 0.0270478\n",
      "9 Train MSE: 0.0169484 \tSparsity loss: 0.014174 \tTotal loss: 0.0197832\n",
      "10 Train MSE: 0.0135708 \tSparsity loss: 0.00927507 \tTotal loss: 0.0154258\n",
      "11 Train MSE: 0.0115894 \tSparsity loss: 0.0081538 \tTotal loss: 0.0132201\n",
      "12 Train MSE: 0.0103312 \tSparsity loss: 0.00816472 \tTotal loss: 0.0119641\n",
      "13 Train MSE: 0.00917437 \tSparsity loss: 0.00835424 \tTotal loss: 0.0108452\n",
      "14 Train MSE: 0.0083915 \tSparsity loss: 0.00798041 \tTotal loss: 0.00998758\n",
      "15 Train MSE: 0.00779623 \tSparsity loss: 0.00808132 \tTotal loss: 0.00941249\n",
      "16 Train MSE: 0.00734944 \tSparsity loss: 0.00785454 \tTotal loss: 0.00892035\n",
      "17 Train MSE: 0.00700855 \tSparsity loss: 0.00766381 \tTotal loss: 0.00854131\n",
      "18 Train MSE: 0.00669476 \tSparsity loss: 0.00779145 \tTotal loss: 0.00825305\n",
      "19 Train MSE: 0.00650377 \tSparsity loss: 0.00698666 \tTotal loss: 0.0079011\n",
      "20 Train MSE: 0.00600064 \tSparsity loss: 0.00718371 \tTotal loss: 0.00743738\n",
      "21 Train MSE: 0.00566541 \tSparsity loss: 0.0073112 \tTotal loss: 0.00712765\n",
      "22 Train MSE: 0.00574734 \tSparsity loss: 0.00675132 \tTotal loss: 0.0070976\n",
      "23 Train MSE: 0.00525059 \tSparsity loss: 0.00684258 \tTotal loss: 0.00661911\n",
      "24 Train MSE: 0.00503834 \tSparsity loss: 0.00657567 \tTotal loss: 0.00635348\n",
      "25 Train MSE: 0.00496254 \tSparsity loss: 0.00634394 \tTotal loss: 0.00623133\n",
      "26 Train MSE: 0.00478003 \tSparsity loss: 0.00625404 \tTotal loss: 0.00603084\n",
      "27 Train MSE: 0.00474681 \tSparsity loss: 0.00605509 \tTotal loss: 0.00595783\n",
      "28 Train MSE: 0.00474311 \tSparsity loss: 0.00615628 \tTotal loss: 0.00597437\n",
      "29 Train MSE: 0.00428011 \tSparsity loss: 0.00605045 \tTotal loss: 0.0054902\n",
      "30 Train MSE: 0.00418255 \tSparsity loss: 0.00616588 \tTotal loss: 0.00541573\n",
      "31 Train MSE: 0.00419088 \tSparsity loss: 0.00580817 \tTotal loss: 0.00535251\n",
      "32 Train MSE: 0.00407085 \tSparsity loss: 0.0056494 \tTotal loss: 0.00520073\n",
      "33 Train MSE: 0.00385356 \tSparsity loss: 0.00523736 \tTotal loss: 0.00490103\n",
      "34 Train MSE: 0.00382184 \tSparsity loss: 0.00522718 \tTotal loss: 0.00486727\n",
      "35 Train MSE: 0.00372391 \tSparsity loss: 0.00511245 \tTotal loss: 0.0047464\n",
      "36 Train MSE: 0.00357969 \tSparsity loss: 0.00498948 \tTotal loss: 0.00457758\n",
      "37 Train MSE: 0.00381227 \tSparsity loss: 0.00487638 \tTotal loss: 0.00478755\n",
      "38 Train MSE: 0.00352203 \tSparsity loss: 0.00481535 \tTotal loss: 0.0044851\n",
      "39 Train MSE: 0.00347288 \tSparsity loss: 0.00428925 \tTotal loss: 0.00433073\n",
      "40 Train MSE: 0.00329288 \tSparsity loss: 0.00448029 \tTotal loss: 0.00418894\n",
      "41 Train MSE: 0.00351159 \tSparsity loss: 0.00426345 \tTotal loss: 0.00436428\n",
      "42 Train MSE: 0.00350584 \tSparsity loss: 0.00429598 \tTotal loss: 0.00436504\n",
      "43 Train MSE: 0.00330488 \tSparsity loss: 0.00410366 \tTotal loss: 0.00412561\n",
      "44 Train MSE: 0.00327868 \tSparsity loss: 0.00377962 \tTotal loss: 0.0040346\n",
      "45 Train MSE: 0.00306268 \tSparsity loss: 0.00394039 \tTotal loss: 0.00385076\n",
      "46 Train MSE: 0.00318153 \tSparsity loss: 0.00375002 \tTotal loss: 0.00393153\n",
      "47 Train MSE: 0.00303979 \tSparsity loss: 0.00397687 \tTotal loss: 0.00383517\n",
      "48 Train MSE: 0.00312536 \tSparsity loss: 0.00382304 \tTotal loss: 0.00388996\n",
      "49 Train MSE: 0.00287534 \tSparsity loss: 0.00353072 \tTotal loss: 0.00358149\n",
      "50 Train MSE: 0.00305251 \tSparsity loss: 0.00320351 \tTotal loss: 0.00369321\n",
      "51 Train MSE: 0.00286479 \tSparsity loss: 0.0029645 \tTotal loss: 0.00345769\n",
      "52 Train MSE: 0.00285689 \tSparsity loss: 0.00305719 \tTotal loss: 0.00346833\n",
      "53 Train MSE: 0.00288925 \tSparsity loss: 0.00286612 \tTotal loss: 0.00346248\n",
      "54 Train MSE: 0.00278685 \tSparsity loss: 0.00280555 \tTotal loss: 0.00334796\n",
      "55 Train MSE: 0.00258185 \tSparsity loss: 0.00270222 \tTotal loss: 0.00312229\n",
      "56 Train MSE: 0.00271206 \tSparsity loss: 0.00252305 \tTotal loss: 0.00321667\n",
      "57 Train MSE: 0.00314271 \tSparsity loss: 0.00249987 \tTotal loss: 0.00364268\n",
      "58 Train MSE: 0.0027225 \tSparsity loss: 0.00239151 \tTotal loss: 0.00320081\n",
      "59 Train MSE: 0.00248702 \tSparsity loss: 0.00235511 \tTotal loss: 0.00295804\n",
      "60 Train MSE: 0.00274784 \tSparsity loss: 0.00229175 \tTotal loss: 0.00320619\n",
      "61 Train MSE: 0.0023637 \tSparsity loss: 0.0021467 \tTotal loss: 0.00279304\n",
      "62 Train MSE: 0.00376068 \tSparsity loss: 0.0017723 \tTotal loss: 0.00411514\n",
      "63 Train MSE: 0.00233923 \tSparsity loss: 0.00219192 \tTotal loss: 0.00277762\n",
      "64 Train MSE: 0.00231539 \tSparsity loss: 0.00185233 \tTotal loss: 0.00268585\n",
      "65 Train MSE: 0.0022342 \tSparsity loss: 0.00201337 \tTotal loss: 0.00263687\n",
      "66 Train MSE: 0.00308808 \tSparsity loss: 0.00189729 \tTotal loss: 0.00346754\n",
      "67 Train MSE: 0.0021982 \tSparsity loss: 0.00183677 \tTotal loss: 0.00256555\n",
      "68 Train MSE: 0.00226939 \tSparsity loss: 0.00162599 \tTotal loss: 0.00259459\n",
      "69 Train MSE: 0.00247374 \tSparsity loss: 0.00150849 \tTotal loss: 0.00277544\n",
      "70 Train MSE: 0.00228837 \tSparsity loss: 0.00148242 \tTotal loss: 0.00258486\n",
      "71 Train MSE: 0.0024456 \tSparsity loss: 0.00147809 \tTotal loss: 0.00274121\n",
      "72 Train MSE: 0.00210597 \tSparsity loss: 0.00138623 \tTotal loss: 0.00238321\n",
      "73 Train MSE: 0.0021107 \tSparsity loss: 0.00129582 \tTotal loss: 0.00236987\n",
      "74 Train MSE: 0.0022424 \tSparsity loss: 0.00129065 \tTotal loss: 0.00250053\n",
      "75 Train MSE: 0.00233301 \tSparsity loss: 0.00110327 \tTotal loss: 0.00255366\n",
      "76 Train MSE: 0.00219297 \tSparsity loss: 0.00117603 \tTotal loss: 0.00242817\n",
      "77 Train MSE: 0.00248895 \tSparsity loss: 0.00109521 \tTotal loss: 0.002708\n",
      "78 Train MSE: 0.002521 \tSparsity loss: 0.00096837 \tTotal loss: 0.00271467\n",
      "79 Train MSE: 0.00189789 \tSparsity loss: 0.00103555 \tTotal loss: 0.002105\n",
      "80 Train MSE: 0.00280811 \tSparsity loss: 0.0010167 \tTotal loss: 0.00301145\n",
      "81 Train MSE: 0.00230722 \tSparsity loss: 0.000979726 \tTotal loss: 0.00250317\n",
      "82 Train MSE: 0.00351125 \tSparsity loss: 0.000846072 \tTotal loss: 0.00368046\n",
      "83 Train MSE: 0.00218575 \tSparsity loss: 0.000720168 \tTotal loss: 0.00232979\n",
      "84 Train MSE: 0.00186479 \tSparsity loss: 0.00083085 \tTotal loss: 0.00203096\n",
      "85 Train MSE: 0.00191433 \tSparsity loss: 0.000784804 \tTotal loss: 0.00207129\n",
      "86 Train MSE: 0.00217137 \tSparsity loss: 0.00075687 \tTotal loss: 0.00232275\n",
      "87 Train MSE: 0.00290488 \tSparsity loss: 0.000699203 \tTotal loss: 0.00304472\n",
      "88 Train MSE: 0.0022122 \tSparsity loss: 0.000660279 \tTotal loss: 0.00234425\n",
      "89 Train MSE: 0.00301047 \tSparsity loss: 0.000502598 \tTotal loss: 0.00311099\n",
      "90 Train MSE: 0.00225816 \tSparsity loss: 0.000577241 \tTotal loss: 0.00237361\n",
      "91 Train MSE: 0.00226497 \tSparsity loss: 0.00067302 \tTotal loss: 0.00239957\n",
      "92 Train MSE: 0.00283404 \tSparsity loss: 0.000533305 \tTotal loss: 0.0029407\n",
      "93 Train MSE: 0.00213362 \tSparsity loss: 0.000496102 \tTotal loss: 0.00223284\n",
      "94 Train MSE: 0.0039725 \tSparsity loss: 0.000455502 \tTotal loss: 0.0040636\n",
      "95 Train MSE: 0.00228322 \tSparsity loss: 0.000475096 \tTotal loss: 0.00237824\n",
      "96 Train MSE: 0.00336009 \tSparsity loss: 0.000467131 \tTotal loss: 0.00345352\n",
      "97 Train MSE: 0.00208409 \tSparsity loss: 0.000463994 \tTotal loss: 0.00217689\n",
      "98 Train MSE: 0.00231182 \tSparsity loss: 0.000605519 \tTotal loss: 0.00243292\n",
      "99 Train MSE: 0.00337147 \tSparsity loss: 0.000389955 \tTotal loss: 0.00344946\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 100\n",
    "batch_size = 1000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(train_op, feed_dict={X: X_batch})\n",
    "        mse_val, sparsity_loss_val, loss_val = sess.run([mse_loss, sparsity_loss, loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train MSE:\", mse_val, \"\\tSparsity loss:\", sparsity_loss_val, \"\\tTotal loss:\", loss_val)\n",
    "        saver.save(sess, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_sparse.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa8AAAFsCAYAAAB7FzYbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHsJJREFUeJzt3VuMnWX1x/E19DDtnDvTaQdKLRWsVhOTlqCgXJhqSdCU\nagmJlRAENCVeGGkihCYajQlNNCbWqEEUQUBSegOECxEPaQSkBok0BohWaiuYlnbOnVMPwP9G88eu\n33I/7xy6u6bfz+XK++z33e+e3eX2/bGehrffftsAAMjkvHpfAAAAVdG8AADp0LwAAOnQvAAA6dC8\nAADp0LwAAOnQvAAA6dC8AADp0LwAAOnQvAAA6cyt03mZSYXp1lDvCzhX9fX1ue/znDlz5LHnnef/\n9/Jbb71VVDMz+9e//uVqK1eudLWRkRG5/uTJk67W1NRUVDMzGx8fd7U333yzqNba2ipf8/jx40Xr\no3va0OD/9NX19/X1yfULFiwoOn90T9SIQXVN6jUjbW1tNb/P/PICAKRD8wIApEPzAgCk01CnLVF4\n5oXpxjOvOjly5Ij7Ps+fP18eq57bqOdILS0tcr16ZnXq1ClXU8/WzMxOnDjhaurfwOj5knpfw8PD\n8tjTNTY2yrp6PjQ0NORq0T0pfWYVvaeJiYmi11Sfk5lZc3Ozq6lnltH51bHt7e088wIAzD40LwBA\nOjQvAEA6NC8AQDo0LwBAOvWasAFgllApsijFPDY25mrz5s1zNZUKNNMpQvWaUbJNpeiqXL9K8anr\nV7W5c/U/tyrt197eLo9VSpN9Vd6TuiaVioxeV61XqcTo2JL3zy8vAEA6NC8AQDo0LwBAOjQvAEA6\nBDYATIl6kB8FJhYuXFj0mtHIJTViSY1sqjL2Th0brVfvVZ1fjayKtgRR90oFU6psKaJCHNHILBWk\nKN26xqx8mxkVrDHT4ZYS/PICAKRD8wIApEPzAgCkQ/MCAKRDYAPAlKgH7lG4oHTyQhT4UJM3VDhA\nBSai61LhhGgaxsjIiKupcIe6J9F7ioIUpcep11X3OQpcqNetMmFDTS1R+5Gp48ziaSq18MsLAJAO\nzQsAkA7NCwCQDs0LAJAOzQsAkA5pQwBTMjo66mrRGKjSvZ/UGCgznYwrTfuZ6cTc4OBg8Xp1rihF\nWEqNV1K1KMGprlXd0yhBqepV3uf4+LirqWSj+jsxKx8Zdjp+eQEA0qF5AQDSoXkBANKheQEA0iGw\nAWDaRQ/nVehAhSjUyCgzs+PHj7uaGi8U7cdVus9VdH51/aWBAxXCMNOjrNR1qvce1avsB9be3u5q\n6lqj8VTR2KhSkw288MsLAJAOzQsAkA7NCwCQDs0LAJDOORnY2LNnj6vt2LFDHrts2TJXUw9ob7zx\nRrm+s7OzqAZkpfbTGh4elsd2dHS4mgoHRHtPzZ8/39WeeuopV/vVr34l16sgxqJFi1xt8+bNcn1r\na6urqX2q1NQJVTPT/560tbUVndtMBzFUCCKasDHVCSEqyKEmpKipH2bx3ms1zzupVQAA1BHNCwCQ\nDs0LAJAOzQsAkA7NCwCQTkM0RmWG1eWk//He977X1fbt2zcj51KjVy6//PIZOdd0u+iii2T9zjvv\ndLV3vetdM3w1NU1tRg0m7dixY+77PDAwII9VaUGV1lM1M51C/OIXv+hqf/zjH+X60r2zovOrpPCS\nJUtcTb3/rq4u+ZoqRbh8+XJXO3z4sFyv7kl3d7erLV26VK7fsGFD0fmjMVBRivF00XgqVW9vb6/5\nfeaXFwAgHZoXACAdmhcAIB2aFwAgnXMysPHKK6+42osvviiP/cAHPuBqL730kqtFD4gff/xxVzt4\n8KCrrVy50tX+8Y9/yNcsFT1IPf/8813ttddeK37d7du3u9odd9xRfmEzg8BGnajARjTeSY1IUuOJ\nolFI6m967969rvbyyy/L9Sqc8frrr7taFI7YvXu3qx09etTVVIijr69PvqYarzVv3jxXi/bzWrVq\nlatF4QjlE5/4hKvddNNNrqbCZ2b6s1a1aAyV+pvo6uoisAEAmH1oXgCAdGheAIB0aF4AgHTOycDG\nmaT2sDlw4ICrqcDG/v37p3RuNc3ATAc21PnVg2gzs0cffdTVNm7cWPHqph2BjTo5evSo+z5Hf3sn\nTpxwNRVOiB7ul+79dOzYseL1qqb22DLTe1epYNUll1ziaocOHSq+JhUYifbD+tCHPuRqX/nKV1xN\nBVPMzDZt2uRqN9xwg6tVCWyovqLCMmZ6mklrayuBDQDA7EPzAgCkQ/MCAKRD8wIApEPzAgCkU7YR\nCyZNJWne9773Fa1dvXr1dF+OmelRVr29va724Q9/WK6/6qqrpv2akFeVUUQqhVY6XshMpxVVMjFK\nO6prbWlpcbVotFpjY6Orqf0BFbVHlplOMKrxUuq9m+nRbirBGP27o/bzUp9TdH6VzFSjrKLPdGxs\nzNWi8WDvxC8vAEA6NC8AQDo0LwBAOjQvAEA6BDZmsdHRUVn/zGc+42rqYer3vvc9uT4anYNzk3q4\nXmUUkHpgH42BUuEMFXhQI6fMdBBDhROi9SrwocIJ6jxq3y4zs6GhIVdTo6Ci/by+//3vu5q6f7fc\ncotc39XV5WrqnkTnV/dKrY9GfkVBkFr45QUASIfmBQBIh+YFAEiH5gUASIfAxix2//33y7raK0g9\ntF2xYsV0XxJmoWjvLEWFO6LJC0rp5AcVDDHTEzLU+mifQxXEUOdSx0UhlNLAw65du+T6gwcPutoF\nF1xQVDMza2jwW2dV2eexNJwRTWLp7OwsPtc78csLAJAOzQsAkA7NCwCQDs0LAJAOgY1Z4tVXX3W1\nrVu3Fq9/7rnnXK2np2dK14Rzg3o4Hz3wV1Ma1PpoGoOa7qICI2rqRlRX4Qo14SI6vwpsqPNEU0dU\nkOGll15ytUcffVSuV0GQn/3sZ67W1tYm16vAinrNaJsYFbhRn78Khpjpzy8K3LwTv7wAAOnQvAAA\n6dC8AADp0LwAAOnQvAAA6ZA2nCWeeOIJV4vSTdddd52rvfvd7572a8K5QY03ipJpKoWm0nZR2lCt\nnz9/vqtFo5hUsk7te9fc3CzXKyptp2rReCSVTHzxxRddrbe3V67/9Kc/7WpqtFu0H5faT01da5QA\nVPdfJQijBGqUDK2FX14AgHRoXgCAdGheAIB0aF4AgHQIbCSkghhqdIzau8jMbPv27a4WPSAHaonC\nEVMRPcRXgQsVDolGEan16lxVwgXqXOo7WiVE8rvf/c7VosDHbbfd5moqMBKFaNS/E+r61WgsM/2+\nVIgjuqdV9g57J355AQDSoXkBANKheQEA0qF5AQDSIbCR0L333utqTz/9tKt97nOfk+uZpoHppB64\nRwEgFSRQx1bZO0qFC9TUDzMdelCBi2iaRGlgQx0XTe3YtWuXq/3hD39wNTUZx8xs0aJFsn66KMSi\nwhVVQiyln38UOJlsWIxfXgCAdGheAIB0aF4AgHRoXgCAdGheAIB0GiY7mmOK6nLSbNSePmZml112\nmau1tra62p/+9Ce5fpamDXWUCjNucHDQfZ+rpA3Vv0FR2lCNUlLjiaJknUoRVkk7qmSeuia1d9bL\nL78sX1Olgg8dOuRqas8+M7O1a9e6mrqn6t6blaclq6QC1Wei7lOkp6en5veZX14AgHRoXgCAdGhe\nAIB0aF4AgHQYD3WWGB8fd7XNmzfLY9WYleuvv97VZmkwA2cZFYIYGhqSx6o9oSYmJlwtCheoY6Nw\nRSkVboj23ioNh4yNjbnaXXfdVfyan/3sZ11txYoVcv3IyIirReOtlNL90KL7rK5fBT6i9dEor1r4\n5QUASIfmBQBIh+YFAEiH5gUASIfARh2oh9Gf+tSnXO2vf/2rXL969WpX++Y3vzn1CwMmYXBw0NXU\nxBczHUxSgQkVeDAza29vdzUVYFLniepqakYUIiidEnH77be72v79++Wxa9ascbUtW7a4WlNTk1xf\nOrUkmjqi1qtwRfSZqHtSZY+3yU554pcXACAdmhcAIB2aFwAgHZoXACAdmhcAIB3ShnXQ39/vart3\n7y5e/+CDD7paZ2fnVC4JmDQ18ilK+6lkn0oLRgm0kydPFq2PknWlyTZ1HjOdmBsYGHC1Z5991tXU\nGCczs23btrna0qVLXS0aWaVGQal7UoUa76RqZjoFqfYzq7LHWgl+eQEA0qF5AQDSoXkBANKheQEA\n0iGwMcPUvkaXX3550dqHHnpI1tU4GaBe1IP4KBihQgcqBBE9xFdBABUCifYDK907Khpl9Prrr7va\nTTfd5GpqlNIdd9whX/PKK690NXX90cgtdU/U/Y8CF+q9qvNH66N7fbooBFPlvf7X9RSdFQCAswjN\nCwCQDs0LAJAOzQsAkA6BjRl23333uVq0r8/p1INcs/i/VAfqQU2OKA1GmOkggJraYVZtGkcpFSKJ\n9u1S3+eDBw+6mvqOrl27Vr6mCqeo+zcxMSHXq/sX7f2lqHuqXlMFQ8x0YKa5udnVVHgtOrYEv7wA\nAOnQvAAA6dC8AADp0LwAAOnQvAAA6ZA2nCb79u2T9W984xtn9kKAM6ylpcXVorSeqqtkXJSoLU3a\nNjY2yrpK1qm0YfR9fvzxx11NXb9KQEZ77qnzq+tUqb7oXGrkUjTySiU7o1FQyvDwsKupBGFHR4dc\nPzo66mqMhwIAzEo0LwBAOjQvAEA6NC8AQDoENqbJ008/LevqYaayevVqV4tG5ABnE7V3VUQ9yFfh\ngsHBQbm+dO8vFYIw0+EGFQLZs2dP8XoVDqky8qj0ex7th6VGSaljo3231LEqHBKNp1KBFXVNUdgm\nCtfUwi8vAEA6NC8AQDo0LwBAOjQvAEA6BDbq4CMf+Yir/frXv3Y1AhvIIJr8oIyPj7uaeuBfJVxQ\n5ZrU66qpH9GEkAMHDrjaunXrXO2ee+5xNRUsMdP7oalgSrRHl7pWtfdWdE/V5AsVeImmbqhrVRNC\nIlG4phZ+eQEA0qF5AQDSoXkBANKheQEA0qF5AQDSaVDjTs6AupwUs1rZRk+Ydv39/e77rNJuZnrv\nL5UgjNJqKq2o9n5SCT4zs+7ublcbGBhwtSgBp8YelY6MiqgUnzp/NF5JpQir7Mel7rW6f1H6WaUg\n1WcaXZOqd3R01Pw+88sLAJAOzQsAkA7NCwCQDs0LAJBOvQIbAABMGr+8AADp0LwAAOnQvAAA6dC8\nAADp0LwAAOnQvAAA6dC8AADp0LwAAOnQvAAA6dC8AADp0LwAAOnQvAAA6dC8AADp0LwAAOnQvAAA\n6dC8AADp0LwAAOnQvAAA6dC8AADp0LwAAOnQvAAA6dC8AADp0LwAAOnQvAAA6dC8AADp0LwAAOnQ\nvAAA6dC8AADp0LwAAOnMrdN5367TeTF7NdT7As5V/f397vs8Z84ceezbb/uv/ty5/p+hN998U64/\nfPiwqy1btszVRkdH5fpTp065WktLi6s1NjbK9RMTE6721ltvuZq6/oULFxZfk3rN6J42NPg/fXX9\nvb29cr16/+r88+fPl+vVseozPX78uFyvjm1paan5feaXFwAgHZoXACAdmhcAIJ0G9f9BnwE888J0\n45lXnQwMDLjv83nn6f9drOonT550tej50IkTJ1xNPXNRz4GicynR8yV1/WNjY66mnuNE16SeJQ0O\nDrqaejYVnUv9ux49R1T3Tz0zU+/TzKy5ubnoXPPmzZPr1WfS0dHBMy8AwOxD8wIApEPzAgCkQ/MC\nAKRD8wIApFOvCRsAZgmVootSzOPj466m0nZRKrA07Vcl2bdgwQJXU1MvorpK20VpS0VN7WhtbS1+\nTZXsU2nBaL1KcKpalMBUaUV1n6LzR5M3auGXFwAgHZoXACAdmhcAIB2aFwAgHQIb0+QXv/iFrKut\nGV544QVXu+eee4rP9bWvfc3V1q1b52of+9jHil8TmCz1ID4KTJSOElLBDjMduFC16PxNTU1F53r2\n2Wfl+tdee83VXn31VVfbtWuXq0Xjma699lpXW79+vatdccUVcr0Kx6iaClaY6XtSus2LmQ53qFFQ\nUTAjGhtVC7+8AADp0LwAAOnQvAAA6dC8AADpsJ/XJHzpS19ytR//+Md1uJL/9/73v9/VnnnmGXls\ne3v7TF9OPbCfV50MDw+773OVCRtqn6oosKGCBGpChtrjykyHC37wgx+42sMPPyzXv/HGG66mAgfq\nPUWBCTW54sorr3S1rVu3yvWdnZ1F54+oCR9KNCFDBT6Gh4eLjjPT01S6u7vZzwsAMPvQvAAA6dC8\nAADp0LwAAOnQvAAA6ZA2rGEmkoVr1qxxNTUiZt++fXL9z3/+86LzRCOnbrnllqL1yZA2rJOjR4+6\n77Ma2WSmk2VqlFO0d1SpKNl31113udp3v/tdV4tGFql9ti699FJXU+Pa+vv75Wv+5je/cTWV1vvW\nt74l11922WWuppKBKpVppvf+UvevymeiUp3RHm3qukgbAgBmJZoXACAdmhcAIB2aFwAgHfbz+rd/\n/vOfsv7Tn/60aL16aGpm9uSTT7qaGpOiHnBH++f8/e9/dzW1/1Bvb69cD0wn9Xc6NjYmjz116pSr\nqXBENEpI7Y+nQmdqjy0zs8cee8zVVAjj6quvlutV4GPRokWuFgVWFPU93bNnj6u98sorcr0KgKnx\nWNEYKHX96nNSNbN4bNTpohDMZMM5/PICAKRD8wIApEPzAgCkQ/MCAKRDYOPfonCDehiswhnqv5I3\nq7avzunuv/9+WX/++eeL1m/cuHHS5wZKNTc3u1oU2FB7yR0/ftzVorCSmsagpjmoqR1mOgii9s7a\nvn27XL948WJXU5Mj1DU98sgj8jX/8pe/uJp6n+vXr5fru7q6XK3KhAx1r9WxahKHmQ7RqMBKFPiI\nPuta+OUFAEiH5gUASIfmBQBIh+YFAEiH5gUASIe04b+tXbtW1lUKUSVpFi5cOO3XFI2mUkkmoF5U\nsi9KlqlkWrTPVCmV6P3gBz8oj925c6erqURxNJ5Kva/StOQzzzwjX/PIkSOudvHFF7taT0+PXK8S\nnCotqO69mR4lpa4/2vtRfX7q2GiPNdKGAIBzBs0LAJAOzQsAkA7NCwCQDoGNGtTD0Jnw4IMPutre\nvXuL11911VWuph76AtNNBTY6OjrksWpslFof7RGl6uo1o/FUpXtXReECpa2tzdV+8pOfuFo01k0F\nHpYtW+Zq0ag5da0qVBatHxkZcTV1/6LPVJ1fBTaikV0ENgAA5wyaFwAgHZoXACAdmhcAIB0CG3Xw\n5z//2dW2bNniauq/3DczO//8811tx44drjZv3rxJXB1QzcTEhKtFez+pIIF6YB893FdTItT5x8fH\n5XoVzlDhgigwoibp/P73v3e1H/7wh0VrzcyWLFniatdcc42rqX3DzPQ9Udcf3dOBgQFXU/dU7dtm\npu+pOn90/dHfSi388gIApEPzAgCkQ/MCAKRD8wIApEPzAgCkQ9qwDp577jlXi5KFyq233upqq1at\nmtI1AZOlxgNF45VKk2nRnnVRYu100R5hpclGNfLJTKfw/va3v7nasWPHXC1K1W3cuNHV1qxZ42rR\nyCt1rSpBqa7dTN9rtV6NkTLTY6fUa0ZjoNR1tba2ymPfiV9eAIB0aF4AgHRoXgCAdGheAIB0CGzM\nsJtvvtnVHnnkkaK1t912m6zffvvtU7omYDqVPrA30yPL1LEq2GFmNmfOnKLa3Ln6nzZ1rAqBROGI\n73znO662c+dOV1OBleuuu06+5ic/+UlXU+8/Gs+k3qsKXKgxUhH1mlVCOOpzrrJHWgl+eQEA0qF5\nAQDSoXkBANKheQEA0iGwMU2i//r8l7/8paup/6J86dKlrrZt2zb5mmpPJKBe1DSJaO8oNTlBhSOi\nh/sqCKACF9HeWWq9mvDR19cn1x84cMDVVqxY4WpqmsaGDRvka6ogRpUJFepc6p5E+/upe6WCLdFn\nql5XHate08ysqalJ1mvhlxcAIB2aFwAgHZoXACAdmhcAIB0CG9Mk+q/njxw5UrT+y1/+sqt1dnZO\n6ZqAM6HKNIbScEe0pYl66K8CF9GEDRV6UAGqb3/723K9CnL09PS42vr1613twgsvlK9ZGs5QoS4z\nvZ2Sun/RligqMDIwMOBqUVCsdJub6G9CBXaiz/+/zlHzCAAAzjI0LwBAOjQvAEA6NC8AQDo0LwBA\nOqQNJ+GFF15wtd27dxev37Rpk6tt3bp1KpcE1I1KlkXjmRSVglP7UZnpZKIajxTtJ6Y8//zzrvbb\n3/5WHrt8+XJXW7lypavdcMMNrhal7VRasKury9Wi8VDqXo2Pj7taNB5K7fOlzhXtsaZeV72naL36\n+ynBLy8AQDo0LwBAOjQvAEA6NC8AQDoENmpQDz7vvPNOV6vygPjSSy91NfbowmwSfR9U4KK0Zqb3\nzVP7QVVZ//Wvf93VqowyWrNmjTz2dFEwQQUehoeHXU0FU8z0eCcVjogCG+r+qRBI9G+UuldqjNd0\n/xvHLy8AQDo0LwBAOjQvAEA6NC8AQDoENmq4++67XS36r++Vm2++2dWYpoHZRIUDSvZj+g8VDoj2\n41LhAhWEOHnypFz/wAMPuJqamBPtnXX11Ve72ubNm11NvafomtT1q8BLNLVEBSbUhIxoQoeqq8+v\nyn5e6lg1dWMq+OUFAEiH5gUASIfmBQBIh+YFAEiH5gUASKch2jdnhtXlpJOhEj5VRkENDQ25WktL\ny5SuCZKeB4QZNzw87L7PUVpQ/XsTjWJS1LFqFFT0mh/96Edd7dChQ6524YUXyvV79uxxNZWAPHbs\nmKtF45kGBwddTY2hUvuemek9xtR9VqOxzMxaW1tdTd3T6PrVsWq8VZVe093dXfP7zC8vAEA6NC8A\nQDo0LwBAOjQvAEA6jIeaYeohabSvz1SovX7UnjpmehxMldEtao+zHTt2FK9X1LVu27ZNHhs9OEZ9\nqFFAKnBgpv9O1bHReCn10F8FqKJRRupv5z3veY+rqb9xM7M33njD1Uq/e9HfbWmAq7u7W9ZVYKK/\nv9/VonvS19fnak899ZSrRYER9e+J+kyvv/56ub6rq0vWa+GXFwAgHZoXACAdmhcAIB2aFwAgHQIb\nM2zZsmVn5Dy33nqrq11wwQXy2MOHD7vaj370o2m/pqmK7t0XvvCFM3wl+F9GR0ddLQpcqD2tVAhD\nvaaZ2aJFi1xNBSGiwIgKd6iwUrR31rp161xNvdf29nZX6+jokK+pvo9qEsiSJUvkehUuuffee11t\n8eLFcn1zc7OrqQkZ6j2Zme3fv9/V1GeyatUquf7jH/+4rNfCLy8AQDo0LwBAOjQvAEA6NC8AQDo0\nLwBAOqQNa1AjTe677746XMn/dvfdd0/7a0Z7MkVjp073+c9/XtavuOKKovUqcYWzjxo7FI0bU387\nqhbtx6XSilX2ntqwYYOrPfTQQ0WvaabTiioB2dPT42rRflYHDx50NTVWLlqvrlXd04GBAblejXe6\n6KKLXG316tVy/Y033uhq6m9CjeGKji3BLy8AQDo0LwBAOjQvAEA6NC8AQDoN0UPAGVaXk06XBx54\nwNXUg9wq9u7d62pTHdn01a9+VdYvueSSovXXXHONrEdjaupMP2HHjBsZGXHf5yrfh1OnTrlaFBZS\ne+GpcEY0Xkq97pNPPulqao8rM7Pe3l5XU4GHwcFBV9u5c6d8TbVPVltbm6tt2bJFrm9tbXU1Nd5p\n06ZNcv3y5ctdTb2nKvvoqc802g9MBTa6u7trfp/55QUASIfmBQBIh+YFAEiH5gUASIfABmYLAht1\ncujQIfd9jh7uq8CFClFEEy7UsWoah5pQYWbW0tLiatHeXYrae0u9J/Wa6jgzff0q8BK9J7WfmNo7\nTE0nMdN7n6nrj0I4KnDR2NjoakNDQ3K92k+spaWFwAYAYPaheQEA0qF5AQDSoXkBANKheQEA0iFt\niNmCtGGdjI6Ouu9ztJ+XqqtRSNGecerfKzXKKBovVZrsi9KS0T5jJeeP0nbqNVUqUr1PM53MVK8Z\n7Zul1qu04Pj4uFyvUpBNTU2uFn0m6nUXL15M2hAAMPvQvAAA6dC8AADp0LwAAOnoJ2gAUEiNF4qo\nvadUCGNgYECuV+ECFW6IghVqnykVGIhGKam6un4VTogCEyqIoV5TXbuZDreoY6uMd1L3L9qPS4U7\n1DVF47GiIEct/PICAKRD8wIApEPzAgCkQ/MCAKRDYAPAlKiH89GEDBWOUPs5Rft5lQYuoj261ISP\n6FpLqWkS6jqjqSMqsKDCFSqYYqYnXKhwRRSMUO9fBUaiEIwKfETTQKYTv7wAAOnQvAAA6dC8AADp\n0LwAAOnQvAAA6ZA2BDAlauxPNIpIJfNUCm/BggVyvRpFpZKFKoFnZtbR0eFqw8PDrlYlWaeShSrB\nF6X91Ho1cikaWaXuv0omRns3qveq7l/0maj3qq41SpBONu3JLy8AQDo0LwBAOjQvAEA6NC8AQDoN\n0UM8AADOVvzyAgCkQ/MCAKRD8wIApEPzAgCkQ/MCAKRD8wIApEPzAgCkQ/MCAKRD8wIApEPzAgCk\nQ/MCAKRD8wIApEPzAgCkQ/MCAKRD8wIApEPzAgCkQ/MCAKRD8wIApEPzAgCkQ/MCAKRD8wIApEPz\nAgCkQ/MCAKRD8wIApEPzAgCkQ/MCAKTzf6gyd5oBjLZ5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7af96c0b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_reconstructed_digits(X, outputs, \"./my_model_sparse.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variational Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "n_inputs = 28 * 28\n",
    "n_hidden1 = 500\n",
    "n_hidden2 = 500\n",
    "n_hidden3 = 20   # latent coding\n",
    "n_hidden4 = n_hidden2\n",
    "n_hidden5 = n_hidden1\n",
    "n_outputs = n_inputs\n",
    "\n",
    "learning_rate = 1e-3\n",
    "\n",
    "dense_layer = partial(tf.layers.dense, activation=tf.nn.elu, \n",
    "                      kernel_initializer=tf.contrib.layers.variance_scaling_initializer())\n",
    "\n",
    "X = tf.placeholder(tf.float32, [None, n_inputs])\n",
    "\n",
    "h1 = dense_layer(X, n_hidden1)\n",
    "h2 = dense_layer(h1, n_hidden2)\n",
    "h3_mean = dense_layer(h2, n_hidden3, activation=None)\n",
    "h3_log_gamma = dense_layer(h2, n_hidden3, activation=None)\n",
    "noise = tf.random_normal(tf.shape(h3_mean), dtype=tf.float32)\n",
    "h3 = h3_mean + noise * tf.exp(0.5 * h3_log_gamma)\n",
    "h4 = dense_layer(h3, n_hidden4)\n",
    "h5 = dense_layer(h4, n_hidden5)\n",
    "logits = dense_layer(h5, n_outputs, activation=None)\n",
    "outputs = tf.sigmoid(logits)\n",
    "\n",
    "reconstruction_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=X))\n",
    "latent_loss = 0.5 * tf.reduce_sum(tf.exp(h3_log_gamma) + tf.square(h3_mean) - 1 - h3_log_gamma)\n",
    "loss = reconstruction_loss + latent_loss\n",
    "\n",
    "train_op = tf.train.AdamOptimizer(learning_rate).minimize(loss)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Train cost: 17875.2 \tReconstruction loss: 14240.6 \tLatent loss: 3634.61\n",
      "19% Train cost: 17548.1 \tReconstruction loss: 13777.4 \tLatent loss: 3770.78\n",
      "2 Train cost: 17034.8 \tReconstruction loss: 13168.9 \tLatent loss: 3865.9\n",
      "39% Train cost: 16482.2 \tReconstruction loss: 12694.2 \tLatent loss: 3788.05\n",
      "49% Train cost: 16032.7 \tReconstruction loss: 12276.8 \tLatent loss: 3755.82\n",
      "5 Train cost: 15633.8 \tReconstruction loss: 11945.7 \tLatent loss: 3688.03\n",
      "6 Train cost: 15861.8 \tReconstruction loss: 12043.6 \tLatent loss: 3818.26\n",
      "7 Train cost: 16033.8 \tReconstruction loss: 12243.4 \tLatent loss: 3790.37\n",
      "8 Train cost: 15669.4 \tReconstruction loss: 11919.0 \tLatent loss: 3750.39\n",
      "9 Train cost: 15610.7 \tReconstruction loss: 11797.2 \tLatent loss: 3813.54\n",
      "10 Train cost: 15734.4 \tReconstruction loss: 12052.3 \tLatent loss: 3682.09\n",
      "11 Train cost: 15623.2 \tReconstruction loss: 11804.1 \tLatent loss: 3819.05\n",
      "12 Train cost: 15977.3 \tReconstruction loss: 12092.2 \tLatent loss: 3885.17\n",
      "13 Train cost: 15843.5 \tReconstruction loss: 12004.9 \tLatent loss: 3838.61\n",
      "14 Train cost: 15393.1 \tReconstruction loss: 11578.9 \tLatent loss: 3814.25\n",
      "15 Train cost: 15663.0 \tReconstruction loss: 11889.6 \tLatent loss: 3773.32\n",
      "16 Train cost: 15253.1 \tReconstruction loss: 11408.1 \tLatent loss: 3844.96\n",
      "17 Train cost: 14680.3 \tReconstruction loss: 11130.2 \tLatent loss: 3550.09\n",
      "18 Train cost: 15194.4 \tReconstruction loss: 11436.9 \tLatent loss: 3757.49\n",
      "19 Train cost: 15213.5 \tReconstruction loss: 11517.4 \tLatent loss: 3696.06\n",
      "20% Train cost: 15504.4 \tReconstruction loss: 11723.2 \tLatent loss: 3781.17\n",
      "21 Train cost: 14657.8 \tReconstruction loss: 10954.8 \tLatent loss: 3703.0\n",
      "22 Train cost: 14923.6 \tReconstruction loss: 11239.8 \tLatent loss: 3683.8\n",
      "23 Train cost: 15046.8 \tReconstruction loss: 11250.5 \tLatent loss: 3796.34\n",
      "24 Train cost: 14317.8 \tReconstruction loss: 10749.9 \tLatent loss: 3567.96\n",
      "25 Train cost: 15251.5 \tReconstruction loss: 11554.8 \tLatent loss: 3696.7\n",
      "26 Train cost: 14774.3 \tReconstruction loss: 11169.2 \tLatent loss: 3605.05\n",
      "27 Train cost: 15071.6 \tReconstruction loss: 11280.5 \tLatent loss: 3791.07\n",
      "28 Train cost: 14530.7 \tReconstruction loss: 10921.0 \tLatent loss: 3609.75\n",
      "29% Train cost: 14655.2 \tReconstruction loss: 10987.5 \tLatent loss: 3667.69\n",
      "30% Train cost: 14957.4 \tReconstruction loss: 11297.5 \tLatent loss: 3659.91\n",
      "31 Train cost: 14998.9 \tReconstruction loss: 11261.2 \tLatent loss: 3737.73\n",
      "32 Train cost: 14851.5 \tReconstruction loss: 11157.3 \tLatent loss: 3694.14\n",
      "33 Train cost: 14992.6 \tReconstruction loss: 11296.3 \tLatent loss: 3696.26\n",
      "34 Train cost: 14449.3 \tReconstruction loss: 10744.7 \tLatent loss: 3704.64\n",
      "35 Train cost: 15177.0 \tReconstruction loss: 11381.1 \tLatent loss: 3795.89\n",
      "36 Train cost: 15058.1 \tReconstruction loss: 11355.4 \tLatent loss: 3702.67\n",
      "37 Train cost: 14821.3 \tReconstruction loss: 11129.8 \tLatent loss: 3691.44\n",
      "38 Train cost: 14752.2 \tReconstruction loss: 11011.3 \tLatent loss: 3740.84\n",
      "39 Train cost: 14596.8 \tReconstruction loss: 10949.4 \tLatent loss: 3647.42\n",
      "40 Train cost: 14694.0 \tReconstruction loss: 10995.9 \tLatent loss: 3698.13\n",
      "41 Train cost: 14499.3 \tReconstruction loss: 10767.7 \tLatent loss: 3731.55\n",
      "42 Train cost: 14830.7 \tReconstruction loss: 11197.3 \tLatent loss: 3633.34\n",
      "43% Train cost: 14656.4 \tReconstruction loss: 10998.3 \tLatent loss: 3658.07\n",
      "44 Train cost: 14954.2 \tReconstruction loss: 11257.9 \tLatent loss: 3696.24\n",
      "45 Train cost: 15002.2 \tReconstruction loss: 11240.9 \tLatent loss: 3761.3\n",
      "46 Train cost: 14050.2 \tReconstruction loss: 10452.3 \tLatent loss: 3597.89\n",
      "47% Train cost: 14473.6 \tReconstruction loss: 10740.6 \tLatent loss: 3733.02\n",
      "48 Train cost: 14555.2 \tReconstruction loss: 10851.3 \tLatent loss: 3703.93\n",
      "49 Train cost: 14944.6 \tReconstruction loss: 11207.9 \tLatent loss: 3736.65\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 50\n",
    "batch_size = 150\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        n_batches = mnist.train.num_examples // batch_size\n",
    "        for iteration in range(n_batches):\n",
    "            print(\"\\r{}%\".format(100 * iteration // n_batches), end=\"\")\n",
    "            sys.stdout.flush()\n",
    "            X_batch, y_batch = mnist.train.next_batch(batch_size)\n",
    "            sess.run(train_op, feed_dict={X: X_batch})\n",
    "        cost_val, reconstruction_loss_val, latent_loss_val = sess.run([loss, reconstruction_loss, latent_loss], feed_dict={X: X_batch})\n",
    "        print(\"\\r{}\".format(epoch), \"Train cost:\", cost_val, \"\\tReconstruction loss:\", reconstruction_loss_val, \"\\tLatent loss:\", latent_loss_val)\n",
    "        saver.save(sess, \"./my_model_variational.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_variational.ckpt\n"
     ]
    }
   ],
   "source": [
    "# encode\n",
    "n_digits = 3\n",
    "X_test, y_test = mnist.test.next_batch(batch_size)\n",
    "codings = h3\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    codings_val = codings.eval(feed_dict={X: X_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_variational.ckpt\n"
     ]
    }
   ],
   "source": [
    "# decode\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_val})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAG9CAYAAADQupsnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGtxJREFUeJzt3XmMXXX9N/DTfS9QaSuoKDsEERGaEgUhPy0NsimFyC5L\nwaBEEpEIKDsIIqBQEhYJhFCEyCJRAaFQQIOtYRE0BaoUKChLm4a2dl/o758nz/N8vufameln5s7M\nndfrv/flnnNPC2fenPuZ7zn9NmzYUAFARv/uPgAAej9lAkCaMgEgTZkAkKZMAEhTJgCkKRMA0pQJ\nAGnKBIC0gd30uZbdt7Z+3X0ANJXzubW163x2ZQJAmjIBIE2ZAJCmTABIUyYApCkTANKUCQBpygSA\nNGUCQJoyASBNmQCQpkwASFMmAKQpEwDSlAkAacoEgDRlAkCaMgEgTZkAkKZMAEgb2N0HANCbbNiw\nIeR+/fp105H0LK5MAEhTJgCkKRMA0pQJAGl9agC/fv36kD/44IPae+6+++6QX3311ZDvuOOOkNsz\nfDv44IND3nPPPWvv2X///UPed999Qx4yZEibnwOtohxyV1VVrV69OuSlS5eG/NZbb9W2GTlyZMgr\nV64MefHixSH/4Q9/qO1j1qxZIb/xxhshf+ITn6ht89WvfjXk73//+yFvueWWIbfCEN+VCQBpygSA\nNGUCQFq/Rt9NNkFTPnTJkiUhH3/88SE/8sgjbe7jsMMOC3m77bbr8HFcf/31Ibfn7/zYY48N+fbb\nbw950KBBHT6OJur9XwDTEZ1+Pq9Zs6b22iuvvBLyFVdcEfLcuXNr24wfP36j+y3PxXfffbe2j3K2\numrVqpAbzTuGDh0a8vbbbx/yQw89FPKnP/3p2j56kHadz65MAEhTJgCkKRMA0lp6ncns2bNDfuyx\nx0K++OKLa9scddRRIe+0004h9+/f8f696qqrNnpcVVVV06ZNC/lXv/pVyOV3u9OnT+/wcUBvMWDA\ngNprW2yxRcjDhg0L+WMf+1htm3J91h577BHyxIkTQy7XojU6lnJmUs5mq6qq7rzzzpDnzJkT8ve+\n972Q77///to+evhctMaVCQBpygSANGUCQJoyASCtpRct3nfffSGXN3EsFw51p3/9618hf/GLXwx5\n3bp1ITdaXNWDWLTYt3T6+dzo51I5HF++fHnI5Y0fq6p+o8cRI0aEXP5CTaMFiG3dhPGjjz6qvbZg\nwYKQTznllJDff//9kGfOnFnbx5gxYzb6uU1k0SIAzaFMAEhTJgCktfTMpJwzlH/WnrwoqLwp5b33\n3hvyc889V9um0UO3uomZSd/SLT9ESu35WdYVD6Fq9LnlDSUvuOCCkB988MGQf/nLX9b2ccABB4Tc\njQ/QMjMBoDmUCQBpygSAtJa+0ePAga3zxyu/l2308CDoy7prptDoc8ubQ44dOzbkFStWhPzXv/61\nto9yZtLTuTIBIE2ZAJCmTABIa52hQi9XzkTKNTLl97KtNA+CVrdw4cKQy/t5lf+8N3JlAkCaMgEg\nTZkAkKZMAEgzxe0hVq1aFfLzzz8f8rhx40Lea6+9uvyYgLY1utFj+Qs0Tz31VMiLFi0KuXxIV2/U\n+/8EAHQ7ZQJAmjIBIM3MpId49dVXQ37zzTdDHj9+fDMPB0hYvHhxyOX5XS467kEPtttkrkwASFMm\nAKQpEwDSzEx6iKeffrq7DwHYBI3WmUyfPj3k8mFYn/3sZ0M++OCDa/vorod9bSpXJgCkKRMA0pQJ\nAGnKBIA0A/heYuLEid19CEADK1eurL12zTXXhDxkyJCQb7zxxpCHDh3a+QfWZK5MAEhTJgCkKRMA\n0vo1WnDTBN3yoT1F+SCsqqqqrbfeOuTy38usWbNC3mWXXTr/wDpP71ptRVafOp/Lc/N3v/td7T2H\nH354yAcccEDIM2bMCLm88WMP067z2ZUJAGnKBIA0ZQJAWo/+oq5V/eAHP6i9tmTJkpBPOumkkHv4\njAT6jDVr1oR8xhln1N5T3qTxkksuCbmHz0g2iSsTANKUCQBpygSAtNb74q4HWr9+fcj/+Mc/2tzm\nO9/5TlcdDtAB5bqSZ599NuT33nuvts0222wT8oQJEzr/wHoYVyYApCkTANKUCQBpygSANAP4Jrjs\nsstCfvLJJ2vvmTRpUsi77rprlx4T0D4rVqwI+bzzzgu5XKBYVVV18sknh9wKD79qiysTANKUCQBp\nygSANA/H6gJr164NecSIESGXixirqqruvffekI866qjOP7Dm8XCsvqVlzuePPvqo9tpDDz0U8umn\nnx7yqFGjatu8/PLLIY8ePboTjq7beDgWAM2hTABIUyYApFln0gVuu+22kMsZSaObOB5xxBFdekxA\nXTkjWbhwYe09119/fcjlnPnII4+sbTNy5MhOOLrexZUJAGnKBIA0ZQJAmjIBIM2ixQ4qh+lPP/10\n7T2HHXZYyKtXrw557ty5tW223377/MH1HBYt9i295nwuf96VN3G86KKLatvcddddIe+yyy4h//rX\nv65tM378+E09xJ7IokUAmkOZAJCmTABIs2ixg37zm9+E/M1vfrPNbW655ZaQW2w+Ar3GmjVrQn7m\nmWdCvueee9rcZurUqSFvueWWnXR0vZsrEwDSlAkAacoEgDQzky6w5557hnzKKad005FA31auK1m5\ncmXI8+fPD3no0KG1fRxzzDEhT5kyJeQBAwZkDrFluDIBIE2ZAJCmTABIc28uuoJ7c/UtveZ8Ln/e\nlffaK3NVVdXgwYND7tevz/3n7d5cADSHMgEgTZkAkKZMAEgzgKcr9LkJZR/nfG5tBvAANIcyASBN\nmQCQ1l03evSdOrQO5zOuTADIUyYApCkTANKUCQBpygSANGUCQJoyASBNmQCQpkwASFMmAKQpEwDS\nlAkAacoEgDRlAkCaMgEgTZkAkKZMAEhTJgCkKRMA0pQJAGnKBIA0ZQJAmjIBIG1gN33uhm76XJqj\nX3cfAE3lfG5t7TqfXZkAkKZMAEhTJgCkKRMA0pQJAGnKBIA0ZQJAmjIBIE2ZAJCmTABIUyYApCkT\nANKUCQBpygSANGUCQJoyASBNmQCQpkwASFMmAKQpEwDSBnb3AQD0ZBs2bNho/uijj2rb9OvXL+T+\n/ftv9J+3AlcmAKQpEwDSlAkAacoEgDQD+A5auXJlyOeee27tPdOmTQu5HNjddNNNtW2OOOKIkMeN\nG7ephwi005o1a0J+4403au+54YYbQn700UdDXrJkSW2bkSNHhnzLLbeEfOCBB4Y8YMCAtg+2h3Nl\nAkCaMgEgTZkAkNav/D6/SbrlQzfFP//5z5AnT54c8vz589vcx9ChQ0NetWpV7T177713yM8888xG\n99HDtd6KLDam15zPixYtCvmss84K+dlnn61tU85Jy3nI4sWLa9ssXbo05DFjxoT88ssvhzx+/Pj/\ncsQ9QrvOZ1cmAKQpEwDSlAkAadaZtGHWrFkht2dGcscdd4S85557hjx16tTaNs8//3zIF1xwQciX\nXnppyMOGDWvzOKCvW7FiRcj77rtvyPPmzQu5nG1UVVUdffTRIU+ZMiXkG2+8sbbNgw8+GPKHH34Y\n8ksvvRRyue6kqnrfzSBdmQCQpkwASFMmAKQpEwDSDOAL5QKkCy+8cKPvP/PMM2uvHXfccSGXN3F7\n4oknattsvvnmIV933XUhr127NuRf/OIXGz0u6GsaPfHw2GOPDblchFwuBi5/0aWqqurEE08MeeDA\n+GNz2bJltW0eeeSRkFevXh3yn//855AnTZpU24cBPAB9jjIBIE2ZAJBmZlIobwT3zjvvhHzYYYeF\nfPnll9f20daDbhYuXNjh4yofuGVmAlF5Q8aqanzjxv/foYceGnI5H6mqqhoyZMhG97F8+fLaa23d\nQPeVV14JudG8p3//3vX/+r3raAHokZQJAGnKBIA0M5NC+Xvm5e+U77fffiGPGjWqw59x33331V5r\n6zvWrbfeusOfA31Jo1lkOUcpz9dzzjkn5EbzkXK9x/r160N+7bXXatuUM5Ayl8fVaGbS27gyASBN\nmQCQpkwASDMzKdx1110hlzOTHXfcscP7XLp0acg33HBD7T1t3Yfn+OOP7/DnQl9S3nerquqzyM02\n2yzkz3zmMyG3535Y5X3yZsyYUXtPeS+uUrkWrbfdh6sRVyYApCkTANKUCQBpygSANAP4NpQLlDZl\n8eDdd98d8vvvv9/hfRxyyCEd3gb6kpdeeqn2WjksL2/KWC4WbGvxcFXVB/0vvvhi7T3lfsubNm6/\n/fYht3Vz2N7AlQkAacoEgDRlAkCamUkHzZ49O+Rddtml9p4rr7wy5J///Ocd/pzyhnPl4iro69pa\nkFhV9VnEsmXLQn700UdDLh9+V1VV9eGHH4ZcPkCr0cOxSuXi529/+9sh97YHYTXS+/8EAHQ7ZQJA\nmjIBIK1fe36vugt0y4e2R/ndZXkDtrFjx4Y8fPjw2j7mz58fcvl33J6buu25554hP//8821u04P0\n/rvW0RE94nyeO3du7bV999035BUrVoRczlkazV3KdWH/+c9/Qi7XojXy8Y9/POS333475EGDBrW5\nj27UrvPZlQkAacoEgDRlAkCaMgEgzaLFwrXXXhvy2WefHfKCBQs6vM9N+SWH8kZxwMbtsMMOtdfO\nPffckC+77LKQFy5cuNHcSPkLNO35hZrJkyeHXC5ibAWuTABIUyYApCkTANJa74u7pDPPPDPk0aNH\nh7xkyZI297HbbruF/OCDD4Z82223tbmPVrjxGzRTowdMnXXWWSEPGzYs5AceeCDkRYsW1fax4447\nhlwuOGz0UK5169aFfNppp4XcnjlLb+MnFgBpygSANGUCQJqZSaG84dqpp56a3ufMmTM7vE2jh/QA\nHVOu5zjjjDNCLmcZa9asqe2jXPN19dVXh/zCCy/UtinnN9ttt13bB9vLuTIBIE2ZAJCmTABIMzPp\nobbddtvuPgRoOeX6jnJG2uieWeXDr8qHYzUycuTIkEeMGNHeQ+y1XJkAkKZMAEhTJgCkKRMA0gzg\nAf6PRjdgLAfwf/nLX0Ju9PC7ctFiOehvRa5MAEhTJgCkKRMA0sxMukGj71hLRx55ZBOOBOioBQsW\ntPmesWPHhtwXHnbX+n9CALqcMgEgTZkAkGZm0g0a/S470DsMHjw45Ebn89ChQ0MuH7DVilyZAJCm\nTABIUyYApCkTANIM4AE6YPTo0SGXN3WsqqpaunRpyKtXrw552LBhnX9g3cyVCQBpygSANGUCQJqZ\nSRMsXry4zfccd9xxIZeLnoDuUc5Eyps2NpqZtGdhY6txZQJAmjIBIE2ZAJBmZtIE5e+YN9IXv2OF\n3mDgwPhj8pxzzgl52rRptW2GDBmy0dyKXJkAkKZMAEhTJgCk9duwYUN3fG63fChNY+DTtzifW1u7\nzmdXJgCkKRMA0pQJAGnKBIA0ZQJAmjIBIE2ZAJCmTABI664bPVrUBq3D+YwrEwDylAkAacoEgDRl\nAkCaMgEgTZkAkKZMAEhTJgCkKRMA0pQJAGnKBIA0ZQJAmjIBIE2ZAJCmTABIUyYApCkTANKUCQBp\nygSANGUCQJoyASBNmQCQpkwASBvYTZ+7oZs+l+bo190HQFM5n1tbu85nVyYApCkTANKUCQBpygSA\nNGUCQJoyASBNmQCQpkwASFMmAKQpEwDSlAkAacoEgDRlAkCaMgEgTZkAkKZMAEjrrodjtYzly5fX\nXvvtb38b8ty5c0O+5JJLatuceuqpIe+0004hn3jiiSGPHDmyto9GrwH/z4YN8Tlea9asCfnVV1+t\nbTN9+vSQ582bF/LSpUtr24wbN26jeYsttgh5ypQptX3svPPOIQ8ePLj2np7ElQkAacoEgDRlAkBa\nv/I7xCbplg/tDOX3o+eee27tPbfeeutG99Ho77xfv34dOo5tttmm9trZZ58d8sknnxzy8OHDO/QZ\nCR37w9Db9djz+aOPPgp50aJFIT/88MMhP/LII7V9vP322yEvWLCgzc/daqutQi7nHe+++27I5Qyl\nqqpq0qRJIZfn92abbVbbpqM/R9qpXTt1ZQJAmjIBIE2ZAJCmTABIM4BvQ7kocf/99w/59ddfr21T\nLh780pe+FPLuu+9e2+bmm28OedmyZRvNjZT/Lm+55ZaQy4WRVdW9AztaRo84nxv9LCvPm0svvTTk\np59+OuRhw4bV9nH44YeH/LWvfS3kctheVVU1cGBcD14eR7mwuVwYWVVV9dZbb4V80EEHhXzdddfV\nthkxYkTttU5gAA9AcygTANKUCQBpZiZtuPrqq0O+4YYbQp41a1Ztm0996lPpz/3ggw9C/uMf/xjy\nT37yk9o2f/vb30Iu5yG33357bZvyBpKdxMykb+kR5/O6detqrz3++OMhX3XVVSGXs42bbrqpto8d\ndtgh5P794/+Db8rcsVxMWd48sqqqaurUqSE/99xzId9zzz21bQ499NCQy2PdRGYmADSHMgEgTZkA\nkGZm0oYBAwaEPHv27JAnTJjQzMP5v1auXFl77X/+539CLr9jHT16dG2bcp3MmDFjOuHozEz6mO75\nIVL87Gr0oLrbbrst5BdffDHkY445JuTy5opVVZ+rdIX169fXXrv88stDvvjii0Pee++9a9uUs9VG\n62Y2gZkJAM2hTABIUyYApCkTANK6frLUy5100kkhN3q6WXdoNFibOXNmyBMnTgx5zpw5tW1OP/30\nkO+///5OODpovvKXZaqqqj7/+c+HXA6699lnnzb30QyNPveEE04I+frrrw+50S9PddfxV5UrEwA6\ngTIBIE2ZAJBmZtKG8sZvb7/9djcdSdvKOUr5EJ9XXnmltk0XPRwLulz5326jecH2228f8rbbbhvy\nqFGjOv/AOsmWW24Z8nbbbRfyrrvuWtumvIFkM7kyASBNmQCQpkwASDMzacPgwYNDLh+U05Mdfvjh\nIV9zzTXddCTQ9RrN/1avXh1yo5ud9lTlDOiTn/xkyOXPpqqqqrVr14Y8dOjQzj+w/8KVCQBpygSA\nNGUCQJqZSQtrz5qYQw45pAlHAp2vvDdVozUW5Vxh+PDhIffkdVblvKetP0tVNedBXv+NKxMA0pQJ\nAGnKBIA0ZQJAmgF8C7vvvvtCLm8cV1VVdfTRRzfrcKBLrVmzpvZaOZAeNGhQyD1lAN/oQVevv/56\nyJtvvnnIEyZMqG3Tv3+8Pij325V/XlcmAKQpEwDSlAkAaS09M1m8eHHI5U3Qxo4d28zD6XLr1q0L\necmSJSF/4xvfqG0zZMiQLj0m6Crlf+/z5s2rvaecEYwZM6bDn9OMuUOjec+MGTNCLs/n3XffvbZN\nOSNq5kzIlQkAacoEgDRlAkBay8xMVq1aVXtt8uTJIf/9738P+fTTT69t89Of/jTk3jRTOP/880N+\n6qmnQr7ggguaeTjQqcrZxZtvvhnyww8/XNtm22233WjelBlD+Z5G27S1n/KmlHPmzKm954knngh5\nr732Cnn8+PG1bcp1Js3kygSANGUCQJoyASBNmQCQ1jID+HKBYlVV1QsvvBDysGHDQr7xxhtr27zz\nzjshH3PMMSGXC/8GDBjQoePcVOvXrw/5wgsvrL3n2muvDfnWW28Neb/99uv8A4MmKRf2nXPOOSGX\nv2BTVVV1/PHHhzxp0qSQy4F1o8F5+Z7ynG/0M6DcT/nLA+VTUE844YTaPsqfaSeddFLIjRZgdueN\nK12ZAJCmTABIUyYApLXMzKQ9rrjiipC32mqr2ntOO+20kB966KGQ77333pDL72Crqv4Qm01Rzkh+\n9KMfhXzNNdfUtjn11FNDLuc9PeVBQNCWRg+L+v3vfx/yn/70p5CHDx9e2+YLX/hCyCNHjgy5PCfK\nxYRVVT8X2zMnLec7H3zwQcg333xzyPPnz6/t4+tf/3rIBx54YMiDBw9u8ziayZUJAGnKBIA0ZQJA\nWp+amZTfMR511FG193zlK18Jedq0aSH/8Ic/DLn8HreqqurMM88MecKECR06zqqqryMpZyQTJ06s\nbVOuK4HeqtHDou65556Qy3UYX/7yl2vb7LPPPiGXN24tZyblfKTRe8q5SqNjLWck06dPD3nmzJkh\nH3nkkbV9/PjHPw553LhxIXfnTR0b6VlHA0CvpEwASFMmAKT1a/T73E3QlA+dMmVKyLNnzw753//+\ndzMOo1q5cmXI5QO4qqqq7rzzzpDL72F/9rOfhXzsscd20tF1CQta+pZOP5+XLVtWe61cM/LWW2+F\nfN1119W2OeOMM0Jua41Io5+H5Yxk3bp1ITf6OVLOd8rZavmgq1NOOaW2jz322CPkZt0HsIF2nc+u\nTABIUyYApCkTANKUCQBpLb1occcddwz50UcfDfmxxx6rbTN58uT05y5fvjzk8hcBZsyYUdtm2223\nDfmBBx4IuVx8Ba2s0U1JV6xYEfLQoUND3myzzdrcT1u/cNTon5cLGd99992Qr7rqqto2c+bMCflz\nn/tcyN/97ndD3mGHHWr76GmLEtvSu44WgB5JmQCQpkwASGvpRYvl4qKdd9455Pfee6+2zfnnnx/y\n2WefHfKgQYNCbnRzxYsuuijkDz/8MOTyAVxVVVWXXnppyGPHjq29pxexaLFv6fTzudHNE6dOnRpy\n+eC6gw46qLbNlVdeGXJ5s8Ty59+CBQtq+3jyySdDfvjhh0N+8803a9uUN50sF0+W89zy50pV9aiH\n2Vm0CEBzKBMA0pQJAGktPTMpletMDj300Da3KX93vXwo1eOPP17bZu+99w65nKE0+m63xfSYL3tp\nik4/nxv9XJo3b17I5cPt3n///do25Zqv8maJm2++ecjDhw+v7WP16tUhjxo1KuQTTjihtk352hZb\nbLHR4+hB85FGzEwAaA5lAkCaMgEgTZkAkNanBvDlE9Ouvfba2nvOO++8kMu/n3JQVi6cqqqqmjRp\nUshDhgzp0HG2gB49TaTTNeV8Xrt2bcivvfZayOXTDKuq/gsy5VMRR4wYEXL5NMeqqqpvfetbIe+2\n224hjx49urbNwIHxHro9fMDeFgN4AJpDmQCQpkwASOtTMxOapld/QUyHOZ9bm5kJAM2hTABIUyYA\npCkTANKUCQBpygSANGUCQJoyASBNmQCQpkwASFMmAKQpEwDSlAkAacoEgDRlAkCaMgEgbWDbb+kS\nHp4ErcP5jCsTAPKUCQBpygSANGUCQJoyASBNmQCQpkwASFMmAKQpEwDSlAkAacoEgDRlAkCaMgEg\nTZkAkKZMAEhTJgCkKRMA0pQJAGnKBIA0ZQJAmjIBIE2ZAJCmTABIUyYApCkTANKUCQBp/wvT5RNB\nWRCAAwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7afdaa518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot the reconstructions\n",
    "fig = plt.figure(figsize=(8, 2.5 * n_digits))\n",
    "for iteration in range(n_digits):\n",
    "    plt.subplot(n_digits, 2, 1 + 2 * iteration)\n",
    "    plot_image(X_test[iteration])\n",
    "    plt.subplot(n_digits, 2, 2 + 2 * iteration)\n",
    "    plot_image(outputs_val[iteration])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_variational.ckpt\n"
     ]
    }
   ],
   "source": [
    "n_rows = 6\n",
    "n_cols = 10\n",
    "n_digits = n_rows * n_cols\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    outputs_val = outputs.eval(feed_dict={codings: codings_rnd})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving figure generated_digits_plot\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEGCAYAAAAwpAFeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXd8VFX+//+czEwmvZNGEhIgEMDQQSC4SlMB1wIINlZX\nXV3Lil91rR/WuhYsq7JiWStYEGFXFxVUpEpvSWiBACEESO+ZPnPv74/5ncMMIAKZCYG9r8fDBxAn\nc889533evehUVUWDBg0aNGhobwg62wvQoEGDBg0aTgRNQGnQoEGDhnYJTUBp0KBBg4Z2CU1AadCg\nQYOGdglNQGnQoEGDhnYJTUBp0KBBg4Z2CU1AadCgQYOGdglNQGnQoEGDhnYJTUBp0KBBg4Z2CU1A\nadCgQYOGdgnD2V7Ar0Drv6RBgwYN5wd0Z/qLmgWlQYMGDRraJTQBpUGDBg0a2iU0AaVBgwYNGtol\nNAGlQYMGDRraJTQBpUGDBg0a2iXaaxafhgDh2AGVOt0ZJ9gEFN7rbK9rPNsQe3S29kdVVXQ6nVyH\nqqooikJLSwtWqxWADh06YDC0fzbTXu6FqqpyX8/mOtoL2j/ltAIakzsKt9uNTqfD4XAAoCgKTqeT\nbdu2sXbtWgD0ej3jx48nOzuboKC2M67FOTmdTlpaWjhy5AhFRUUA5Ofnk5SURHV1NcOGDWPIkCEA\nREdHt/mZut1uuc76+npCQ0MJDQ0lODgYCDyNKYoCQEtLC3PnzuW7776jqamJiIgIAFJTU7n66qsZ\nPnw4YWFhAAQFBQVkXd5MVJyf2Wzmyy+/ZOPGjfTr1w+AyZMnExsb2y7un7cgFet3Op0AVFVVYbVa\n2bJlCxEREfTq1QuAjIwM9Hq939eiKAo2m42qqioAfv75Z0pLS6moqCAiIoKJEycCMHDgQIKDg8/K\n/rndburr64mMjMRkMrX580Fz8WnQoEGDhnaK886CcrvdlJWV8eSTT0rLwGKxcMkll/D000/TqVMn\ngLPidhCam1iny+XC7XZLDVxRFBwOB2azmXfffReAVatW0b17d26++Wb69OlDSEjIGT1baIFCE3K7\n3dhsNgoLC1m4cCEAFRUVFBcX88ILLxAdHQ20jVUg3EHFxcUsXLiQ5cuXc+TIEQAiIiLQ6/XExsbS\n0NCAxWIBYMyYMURERAR8faqq4nQ6WbNmDY888ggA+/btw2w243K5CAkJYdy4cQB88sknZ3w+p7KO\nw4cPA3DjjTeSn5+PzWZDr9cTFxcHQE1NDXFxcfTs2VP+nsFgQK/XH+c20ul0BAUFnbGl7L3vLpcL\ngK+++ooXXniBxsZGNmzYAECvXr0YMmQIRqPxjJ7jLwj3I3hoTlEUXC4X+/fvBzyW+vLly1EUhczM\nTGnZXH/99QQHB/vVihL3/ODBg5SVlQEQGhqK2+3GarUSHBxMbW0tADabTfKHQEPs0fbt2wEYNmwY\nVquVkJAQ1qxZQ9++fdtkHd44rwSU2+3mp59+4t577+XAgQPSJaPT6aRL5Prrrwfgb3/7G4mJiQFz\nZamqKp9vs9k4cuQI8+bNk4e/bds2mpubiY6OJjc3F4D09HQyMzPZvHkz33//PQDNzc0kJiYSGxvr\nl7UKxmIwGIiKimLgwIEsW7YMgAMHDrB48WLKysr44osvAIiMjGz1M38NqqpisVh49dVXAZg9ezYN\nDQ0YjUbJ6KOjo3G5XNTX1/PTTz/x888/A/Ddd9/xwgsv0KFDh4AKKVVVqaysZObMmRw6dAjwnKfD\n4UBVVcxmM0uWLAE8AvaCCy4IyHpaWlq4++67Adi8ebMUjqNGjeLRRx8FoHPnzoSFhWEwGLDZbAA0\nNTXR0tLC4cOH6dChAwAJCQl+c7sJAQ6wZcsWbDYbdrtd0n5VVRVms5nIyMiAuMpOdY1CIQQPnxBr\nFPskFKKkpCRGjBhBjx49AI9C56/zFMqpEI7R0dHyTEJCQrjiiiuora1l6dKl0hVvtVqJjIxsMxef\noigsXrwYALvdjqqqOBwO+Xdo23DJeSWgLBYL77zzDocPH0ZRFMnQhaZos9n46KOPAI/P9x//+Adj\nxozxu3bndDrZtWsXX331FQCrV68mPz+f5uZmabmZTCYSEhLo1asXY8eOBTzaZnV1Nb/88ov8rvDw\ncAYOHEhmZqbf12kwGOjbty9//vOfAWhoaGD79u2sWLGCefPmAXDrrbcGLIbR2NjI/fffLy04i8WC\n0WgkPDxcamsiyO50Olm9ejUVFRUA/PTTTzgcDj744IOAapgiCWDgwIHSetu3bx/V1dW0tLSgKIq8\nuIsXL6Znz55+Z8Qul4t58+axdetWAIxGIx06dOD5559n0qRJki68z0n8zGAwUFxczKJFi6T34KKL\nLiImJsZv6xPCSCgXCQkJMlaYlJRESUkJqampxMfHyzW1FVRVxeVy4XQ65fnpdDoZAxL7EBoaSnp6\nOgkJCXTv3t3Hg+BvAVVfX09ZWRmJiYkyfijOKyIigkmTJrFixQoAKUBbA29eeCprFOsMCgqSZ2u3\n21u9jjPBeSGgxCauWbOG1atX43K5MJlMUgsaOXIk5eXlrFq1isrKSgBKSkq4++67WbZsmby4/rBQ\nrFYrK1as4Mknn2Tv3r0AkpEFBweTmJgIwJAhQ3jkkUfIycmRDE24cSwWi3TFxcbGMnbsWAwGQ8AE\nhWD6zc3N2Gw2FEWhoaHB78/yht1u5/bbb+eHH36QxG80GunZsyfDhw/noosuAiA+Pp7IyEisVitZ\nWVnMnTsXgNLSUlasWMGWLVsYPHgw4J/zOxFiY2MZMmSI/P6IiAgKCgpwuVzSkgHk3vlbQDkcDgoL\nC2XiQ8eOHXn33XfJzc391WeJtYaGhtKpUyeam5spKSkBYOzYsTgcDr8JitLSUsCjiDU1NZGdnc3F\nF18MeM7PaDRSXV1NaGgo4Nm/tkrCEbS8Y8cO+cyoqCgUReHQoUMcOHAA8Fh6ffr0oX///iQkJPj9\nDIW3AODdd9+lc+fOdOrUSZ6B9902mUxERUUBHqXN7Xa3ar9O93e3bNkCHHXd6vV6evXqdVYSNbQk\nCQ0aNGjQ0C5xXlhQQtIvWbJEBhnHjx/P9OnTAY/Pvbi4GIPBIN1JLS0t1NXVsWDBAunbDw0NbZWW\n4Ha7KSws5I033mDnzp3SsouMjCQvL49p06bRv39/wONzFv5t7zTrhQsXsmXLFqlZ3XDDDXTu3Dlg\nGqeiKNTX1wNHffVGozFgbjPxrgsXLuTnn3/GYrFIzfrKK6/kqaeeIiMjQ56Dt3WZmZkpPztr1ixs\nNhsVFRUy+B2IPdLpdISGhtK7d2927doFwMGDB6mtrUVVVUwmk3QHde3aVZ6nP91CVquV/Px8uReP\nP/74Sa2nEyEhIYGsrCzA4zYNDg72i7Wnqipz5swBoLGxEZ1OR5cuXaRVm5KS4pOUAR6a86fr7GRw\nOp0cPnyYwsJC+a7FxcU4HA5KSkro2LEjAFdccQWjRo0iLCwsYGnlInHk4MGD3HzzzcfFlgTtWK1W\nmpqaAE9csa1idyLtXoQYxF2Nj4/3q0v4dHBeCCixkbW1tRgMBiIiIsjJyZHZTTqdjsTERC6//HJ2\n7NgBeBICdDod69atY/LkyYDHddIaYnC73dTW1lJZWUlISIgMgN59991cf/31REdH+3z/sXUkK1as\n4JlnnsFms3HttdcCcNtttwW0BiE4OJiBAwcCHhdNWVkZiqJIZuZvJmI2mwFYsGABDocDvV4vY3D/\n/Oc/iYmJOe7SgmePYmNjGTp0KADz5s2jqamJpKQkKaACFcTV6XRs375dJnOUlZXhdrsxGAzExcVx\n9dVXAx7XWSAYb2VlJfX19fJMRo4cecp06nA4+O6770hOTpZ7ZzKZ/CbMhZAGj4tWp9MxaNAg0tLS\nAKQgFJlrAkJgBUpICVqwWCwsX76cgoICqYg1NjYSHR1NdnY2N998M+CJ/wYHBwdMEXQ6nVJADRw4\nkA4dOhwXUxYKokikAtq0hkxVVTZu3CgzGAVuu+22s5bgcl4IKBHDEH5mvV5PZGSkJNKQkBA6duxI\neHi4zDj64IMP2Lt3L1VVVaxZswaAiRMnSgI9E6LQ6/Xk5uZy00038eOPP0oBNW7cOCIjI09I/Iqi\nyHT466+/nqamJlJTU6X1F+iCVLfbLWMIQrNMSUmRlp6/Iaxdk8lEZGQk6enpPPHEE4AnNnAyBq+q\nKsXFxYAnO81kMlFZWRlQAaWqKjU1NTz88MMcPHgQ8OyZXq8nNTWVV199VaaZB4LBeQf5hSA4ldiR\n2OePP/6YZcuWceONN0qh4c8uBW63W8ZwMzMzsVgs5OTk+CgWIhNMZMq5XC5CQ0NJTEyUFrG/hZXw\nXqxevZr169dTXFwsf5aWlka/fv245ZZbSE1NBQhYjBeOJgRt3LgRgOuuu+6Ez3K73ZSXl7Nu3TrG\njBkDQFhYWJsJKJfLxWOPPSbvE3iStB566KE2ef6JcM4LKO/6EIfDQXBwMJdddhl33323JH5xwGFh\nYYwePRqAH3/8kaKiItxut9QYvA/mTNw0er2epKQkrrnmGrkm8ARgU1JSMBgMPt0t3G43GzZs4Pe/\n/z2A7Arw8ssvk5GRAQQu8A+edywvL5eJB/v370en09GtW7eA1fOIPb7yyivp2bMngwcPlgzOu1bF\nm4kKJr1nzx4fF21ycjLJycnyu/0tmMCTsXfrrbf6BNmDgoLIy8vjrbfeokuXLgGvU9HpdISEhFBX\nVwd46Ck9Pf2EtCGEwaeffgp4BFRmZiZ9+/aV6/QnTSmKIpl8bm4u9fX17N+/nz59+shnORwOvv76\na3l2drud5ORkHnjgAZmt6e9uHGKvPvvsM9atW4fBYJAWnN1u57LLLiMsLKxVCunpwGw2Ex4eDhzd\nE29FQ/Chn3/+mby8PHJycgDa1HLZvXs3RUVFPll8w4cPx2Qy+dVtfTo4LwSUYGoDBw4kOTmZxx9/\nXAonbwQFBcm0zrCwMMLCwlAURRKOd7wDzoxog4ODSU9P58EHH5Qad319PbW1tcTExMhnOBwOPvnk\nE/72t7/R0tICeKyK22+/nSuuuKJNCNPhcPDtt9/KjL2MjAyCgoKIi4sLSGGlqqpy/0ePHk1eXh56\nvV6en81mo7GxEbvdLjOeBGOpqqpi6dKlFBYWAh5tLykpidTUVL8LCFHfBJ5WPdu3b0dRFLn2ESNG\n8Oyzz5KVlRXwAlSdTkd8fDxdu3aVFsiiRYsYN24c8fHxPnU6iqJQWVnJxx9/zKJFiwDP/l1wwQXE\nxcX5naYURcFsNsu2VDabjdDQUPLz82UmWG1tLQcPHqSqqkp6OoKCgrDb7ZSVlfkUf/qLCSqKwocf\nfgh46uUcDoePhSZc+5mZmVx66aVA4AVBVFSUFNrV1dXs3bsXg8Egaf/777+nsLCQpKQkhg0b1uat\nhVRVZcGCBTQ3N8uf6fV6srKysFqtWqsjDRo0aNCgwRvnhQUlsqiee+45YmNjT6pRCz+0iFNlZ2dL\nzcbbb94aN4herychIUFaccXFxZSVlVFSUsLu3bsBeOONN6Q5LdZ77bXX8te//rVNmkO6XC6WL1/O\ngQMHuPLKKwHo0qULu3btIiIiImCWgdBUw8LCMJlMPjVXe/fu5dNPP6WgoEDGxUQHgJSUFJKTk2Vb\npPT0dHr37i3rRcA/Grhwkc2fPx+AXbt2yXiOsKqWL1/O4sWLue+++wJ+Tjqdjg4dOnDTTTfJjhXf\nfPMNH3/8MVFRUXTs2FGuQaw1LS1NurP69+/PyJEj/V4cK/bp/fffl225HA4HCQkJVFdXU1NTA3is\nGaPRSExMDCkpKYAnWzYrK4v09HS/N3QW1u8nn3wCHC10DQsLo1u3boAnJr1s2TLKysoYNGgQAImJ\niQE7S51OR2RkpGxBtWXLFt5++21Z7A2e+kmbzYaqqhQUFNC1a1f5u20BVVVZvXq1zx2KjY1lwoQJ\nhISEHNftXSDQ6zvnBZROpyMpKQnwuDNOJljcbrd0R4gCxvDwcPn7Aq0N2Op0OgwGgzSL6+rqmDdv\nHitWrJBuGuHuMJlMMgYzefLkgLc1ES6F6upqKioqCA8Pl/Gy6Ohompqa6Ny5s088rrXP8/bze7uj\nhCIghIzBYCAsLIyoqCjJYOvq6nwq8L27sYu98nbJ+mPvdDod5eXlwNF4jYiDgadjwvTp02lpaeGv\nf/2rdBEHCnq9nksuuUQK51WrVnHw4EEcDgc6nU4yYVVVpRtZtKiKjo4OSBxTxLlmzpxJY2Mj4Mni\nO3ToEGaz+bhA++WXX85ll10GIIube/To4RN78peAmjdvnhSQBoOBxMREXnjhBS655BIA/v3vf/P8\n88+zc+dOmXQjCugDBaPRKBVhp9PJgQMHiIqKYuTIkYBH4WpoaODDDz/kpZde4vLLLwcC22rMGy6X\ni23btqGqqlRm0tPTyc3NlW5kbwW+rUoFzgsBJTb0ZBdRHIBo4eN2u0lMTKRPnz7H+Vf9dVEE458+\nfTqFhYU4nU6fAKTBYKBDhw50794d8NSMBNJ6UlVVMrO1a9dKK0UIS5fLRUREBBUVFdKCuO6661oV\n4zn2TLzb4jgcDh8G2rVrVx577DHq6+v58ssvAU8yS01NDTExMezfv1+etWiy63a75XcKxaK1yoXR\naGTq1KkALFu2jNWrV8suEeIzDoeDZ599lhkzZvDOO+8AcNNNNwWsFku0pQK46qqrmD9/Pi0tLaiq\nKhMCVFUlMTGRhIQE2WxU1LX5E263m5kzZ/L666/T0NAgBXRSUhLJycnU1dVJpcNut5OTk8M111xD\n586dAWQpSEhIiN+TFISQFvHC6OhoXnjhBa688kppvefm5mI0GrHb7fJzbWGpiD2Jj4/n1ltvpVOn\nTlI5FoqWxWJh+vTpzJ49G4B77rkn4OsCT6NoEX8S93348OHEx8efkKa969oCiXNeQHn3QjuWOQmG\nYrVaWbx4MWvWrJGXtVevXnTp0oW0tLTjtJTWuoqEILjzzjsBZFscOHr4KSkpxMTEYDKZpBbVrVs3\n9Hp9wFoauVwu2c9twYIFGAwGLBaLdJEmJibSuXNntm3bxtKlSwEPg/nDH/7glyCpqqpSGP73v//F\n4XAwfvx46foRKbXR0dEytXXatGlYLBaWLFnCxx9/zJ49ewCPZimEnCgd0Ov1fgl263Q6mZk2a9Ys\nPvroIzZu3CgTN3r27MmqVavYs2cPVqtV9jLMysqSLZr8CdGQVQiijIwM8vLyUBTFJ6W7paWFlJQU\niouLpWWgKIpkjP4ayldXV8fbb79NVVUVwcHBkskOHz6cvn37kpeXJy0Sm82G2WwmPj7e52xEU1t/\nw+Vy0dLSIq2Viy++mLFjx8p6LPAkTtTX1xMfH092drbf13AiOBwO1q9fD3h4QL9+/U7YGGDq1Km8\n//778vzaCp999plUXkX96KOPPvqrZ9RWraq0JAkNGjRo0NAucU5bUCKtVmi2aWlp6HQ6mpqa2L9/\nP6tXrwZg3bp1VFZWkpGRIbsW9O3bl8jISMLDwwOiDaxZs4b8/Hy5TpPJREREBCNGjAA81kpTUxPh\n4eGysWagU5bdbrdsONqhQwecTidxcXFcc801AGRnZxMdHU1ISIiM1b355pv07t2bwYMH+8WyE805\nP/nkExISEhgzZoxP+q/3OBDwWEUGg4Hs7Gw6dOjAtm3bAI+mXFpait1ul21Y/KmRC5pIT0/n3nvv\nxeFwyMLr4OBgGhsbueCCC6ioqJBW4Zw5c8jLy/M7PblcLioqKqT1azKZuOKKK+jatSsxMTGSbmw2\nGy0tLZjNZum+jYyMpK6ujszMzFYX6AoLbP/+/TQ1NUk3j9C8W1paSE9PJyMjQ1rcUVFROJ1OgoKC\npKVrNBplHMM7xuePBBdVVenWrZvsxDJy5EgZ5BfjUj788EMUReG+++6T9yGQUFWVtWvXsmDBAgCe\nfPLJX22rFh4eTk5OjvRqBLr+SLjH3333XdmUdvz48YDHFen97F9LlAgkzlkBJbpwz5kzR5qkPXr0\noKGhgU8//ZSdO3fKDBmLxUJ8fDxDhw4lPT0d8JjcZWVlpKenS7fbsYPdzhQ2m42vv/5aXsjo6GhG\njx7NFVdcId2J1dXVVFVVkZOTI90hgTSbRRxD7FXPnj3p0aMHWVlZ0kUjXDD9+vWTLj6Ab7/9lgED\nBrRaACiKwqpVqwAPkzt8+DD//ve/ue+++4ATC2hVVWlpaWHFihUUFhbKs2poaCAsLIzq6mpiY2Pl\n+v09GqGpqYm9e/fSuXNnn0LXkJAQYmJiZHd88X7+hqIoHDlyhK+++koWT/fo0YPMzExiYmJ8humF\nhoZiNBrR6/UyoeLgwYNERUXJ7hfQ+nhLSkoKo0ePZtWqVYSEhMjstC5dupCSkuIzC008z3s4Ynl5\nOUFBQcTGxsr4lb+UC1VVOXLkiIzBmUwmKeCFcmo2m8nOzuauu+5qk9iT3W6npKSEzMxM4OTti1RV\nJTIyUhagB3p9IqtYJG+Fh4czatQowDd8Iv4Nbefeg3NYQNntdlauXInZbJYZO2VlZZSXl1NZWUls\nbKy8JJ06daJnz57odDrZi0+v12M0GmlqamLAgAEAfqmYVlWV5uZmmpubZRzjnnvuYeLEiTgcDjZt\n2gR4CCMmJsanOWWgs2L0er20NgYPHkxDQ8MJY0s6nU5qm0aj0W9FeqJLBRy1Qt5//325pkmTJhEW\nFobRaJTZegUFBcyaNYs1a9Zgs9kkI4uIiKBHjx4kJyf7jCzwl8YplItFixYxZ84cBgwYwE033QR4\nrJIPPvhAWoNif+64446A9C5ctWoVBQUFkp5GjhxJbGyspFexV0JpKy0tlcpZbGwsW7ZskcyxNRDv\nJsZ9NDU1YTabpWIhhgDW19dLYS6Eo8Vikc12N2zYgM1mY8qUKX7PgHQ4HGzcuFH23VuzZg3l5eW8\n9tpr8rxiYmJYuHBhwLqleEMksRiNRm677Tbg5J6SI0eOkJWVJfuDBhJut1v2lxQZtaGhoT7KWWRk\nJEaj0UfBaMuOEuesgBJdd5ctWybrjSwWi3TFWK1W2S4oOTlZ1vYIoWU0GqmtrSUlJUUyI6fTKQ+j\nNbDb7URFRTF8+HAABgwYgMPhYNu2bXz99deAR0Dl5eWRnJzs0wIlkNDpdD57tWHDBo4cOUK/fv0A\nT8dro9HI7t27pTvJbrfLuVqtRVBQkEwieOedd5g2bRo1NTU8/vjjADz77LOSCQulo7q6WroegoOD\nufDCCwHPOOphw4YRExPjk8Xpr3lMwm316aefsmnTJrZs2SIHUNbX19Pc3CxdVCJ9unfv3n6/vMLq\n3L17t09T4cbGRmJiYti2bZsUUJGRkaxcuZItW7ZIt2OPHj0ICQnxq/Kj1+uJiooiKirKp5OLqFmz\nWCw+2ZY2m43q6mqZJDBv3jwaGxsJDg5m4sSJAKSmpkrrrzXrFOcvnrVkyRIaGxt9Wg395z//kbwh\n0HA6nfz444+/qgyKM3W5XBw6dIiVK1dy+eWXSxdfIFFdXS37A8JR/rB582bAY02NGDEioE19fwta\nkoQGDRo0aGiXOGctKJvNxt69ezlw4IBPfYpwmaWlpclU04SEBGJiYoiMjJQWRFpaGuHh4T5NG0UN\nTGsg0rn79+8vkyQWL17M4cOH2bNnj/T1dujQgbS0NBISEgLebNQb4v369etHY2MjixcvluOlTSYT\ndrudtWvXShff+PHjGTVqlN96lYnnjxkzhq+++oq33npLNhFtbGykpqbmuKaxRqOR+Ph4Ro4cyf33\n3w949i8xMTEgafne6fBNTU1YLBZcLpfseOHtDhk/frzsWhAIl1FoaCjDhw+nvLxcTj7+9ttv+fLL\nL2lubsZgMMhC7/T0dMrKykhJSZFxoUGDBvG73/0uYDSm0+kkbej1eoKDg33cRKKgU9w/8Nxdi8VC\nUVGR9F6IKQT+KJC3WCyUlZXJZ+l0OpKTk3n77bcB/Jbwc6prMhqN7Ny5UzZlHjduHMHBwZjNZnn3\ntmzZwqBBgxg1ahRJSUkBScH3hqIoLFy48LhG2bW1tdI9Gh8f3yZdbU4G3dnIzDgF/Oaidu/eze9/\n/3uZrQQe5texY0fGjh3LqFGjZGwpPDycpqYmbDabZJDh4eEYDAaam5tlkoCYZ9PaA7Hb7Rw8eJD3\n3nsP8Pjcy8rK5DBF8DS2nTZtGhdeeKHfOzmfCoQgLSsr48UXXwQ8risxi0aM23j55ZdJSkoKWDNN\nl8slh7Pl5+ezefNmSkpKZM1PZmYmI0aMoF+/fsTFxZ2wK0UgIATUQw89xJw5c2hubpbPjouLIycn\nhxdffJFBgwYFlJmI4s2SkhIpIMvLyykqKqK8vJyIiAgpoDp27EhjYyMXXnihrC0zmUwBnXN0qu+g\nKIoURps2bWLZsmXk5ubKjgn+Eu6qqlJSUiILXIuLi7nkkkuYPn26dOu19T2rrKxkz5497Ny5E/DM\n97LZbAQHBxMfHw94XLFDhgwhIiKiTdbndrvZtGmTHBUj7mB6errshN+3b99WD3H9/3HGX3DOCqj6\n+np++OEHnn76abmB/fv357bbbqN///6Ehob6MFVxSUQMSqfTYbfbKS8vlwFk787QrYGiKDQ1NcnJ\nlJ988gkbN25EURTJ+B944AEpnM6mhgJH/eCiIPRYrfh/GULZqKyslMykU6dOmEym//m9aQ0CmRHm\nnUWoKMpvtkD7X4XL5eKVV14B4KeffiIqKooJEybIBA0/8qb/PQEFyEmdrUmfDWSdgdhbcVlUVW2z\njD0NGjRoaCf43xRQGjRo0KCh3eOMBZRm92rQoEGDhnYJTUBp0KBBg4Z2CU1AadCgQYOGdglNQGnQ\noEGDhnYJTUBp0KBBg4Z2CU1AadCgQYOGdglNQGnQoEGDhnYJTUBp0KBBg4Z2iXO2WawGDW0NUdRu\ns9nYtWsXTqeTTp06kZCQAPh3YKKG1kGc1cnOQ0zghbYZwncqa9Lgi/NaQHn3mIO2nQTZ3iFaPAV6\npPS5gFNlZqKB7Pz583nrrbeIjIzklltukSOyo6Ki/uf38mxCtD4TE3xPBjHoUTSwNRgMsvdcoM9Q\nCEbvkfcMKkYDAAAgAElEQVTez9Ro6Cg0jq1BgwYNGtolzlsLSoyTAGhubsZoNLZZK/v2iGOtSe8/\nj9XeznYjW+/RDHq9/lddZ6JrtThnMXvndC3lU3lXt9stu9M/8sgjtLS0kJaWRkFBAd27dwcgJyeH\nsLCwgHfodrlc2O12GhoaKCkp4d133wWgsLCQzp078/zzz9OrVy/gf8trIKyioKAgOVbnRB3nxV6K\n/QSPBRVIuvd2J4qpCsLag6P3zuVyYTAY5PrbE7/y7tvaZvO0zsdmsaqqYrVa5djn77//nn79+jF+\n/HiioqKAttlg727mgomK2U+nwzj8QRjigoh1NDQ0sGvXLoxGo9wTl8tFbGwsISEhcuR0W88Scrlc\nrF27lvnz5wNw880306tXr+Na/7vdbnbu3Mkbb7zB8OHDAc9wxdjYWL/PZ1IUhYMHD8rZOQcPHqRD\nhw5MmDCB9PR0OdwR4Pbbb6dbt25+3TNVVTGbzfI5VVVVbNq0idLSUsxmM0uXLgU8I7zdbjcxMTGs\nXLkSgK5du/ptHaeyTkVRsNvtcuid1WolKSmJ8PDwgNKReHZdXR12u12ORhEjdIQ7Gzy043A4fIRG\noBQLOOp6tFgsgIfGm5ubyc/Px2w2Ax7lxul0snfvXoYMGSJHALXlSBexH8KV3dLSQmVlJZWVlRQV\nFcmR9RdeeCFdunQhJCTkVPfsjJnteWdBqapKXV0d9913n5zUajab0ev19OzZk6+++grwXNxACinv\ng164cCGrV69m8ODBjBkzBkAO3zuZZQAegSRGirR2vd4CaufOncyfP5/6+no5rKy8vJyGhgbcbjeX\nXnopADNmzGiz2IqiKOzdu5cnnnhCDqLcv38/t912G3l5ecTExMh1rFu3jokTJ9LU1ER1dTUAV155\npd+ZjBg4d/fdd1NeXg5AZGQko0ePJi0tjfr6evbu3SvXtHHjRt544w169+4N+MeCURSF3bt3S0Wi\nU6dO9OnTRwpiq9UKwOzZs3nqqaeoqanhyy+/BODxxx/3C93A8cqRYGiC8e7Zs4e5c+eyZMkSOdHW\nYDAQGRlJbm4uTz/9NOC5e/5WfIT1sWvXLiwWC926dQM8QxyFNSLuVHNzM1VVVSQnJxMeHg4EztJU\nFEXOWSsqKgI8CsaBAwcwGAxyiGJ8fDx2u52QkBAWLlzIrbfeCiDP/EzwW/FlsS7wKKz79+/nyy+/\n5IcffgCQwl4IeDFU8v777+fOO+9sE55wXgkoVVVpaGhg9OjRFBYWyqFo4CGUgoICLrzwQgBWrlxJ\nz549A0aYTqdTTqr917/+hdFopLa2lh49egAQERFxwksqtKuWlhbAM/I7KirKLwIqKChIMpuamhoq\nKytpbGyUDP7IkSO0tLRgt9v54osvABg2bBg33HCDvOSBgFhTQ0MD//d//8emTZvku5aVlbF161Yu\nuOACQkJC2Lp1KwDXX389NTU1GAwGRowYAUB0dLTfLo1Y06FDhxg1ahQHDhwgLCwMgD59+pCbm0v3\n7t0pKyujtrYW8Gjre/bs4emnn2bmzJkApKamtprGdDodubm5UiB5u4Xg6DTaKVOm8Oyzz6KqKjU1\nNfI9WrMn3sk03j9zOp20tLTwr3/9iw8//BDw0JTVasVut/sIterqakpKSti0aRMAF198MXfccQdD\nhgzxq7UrLJDS0lL5vYmJiVIRFAqj3W4nNTWVkJCQgFsownLasmULW7ZsATwKxujRowkLC5MCUlgj\niqKwYsUK6f0ZNWrUGdPPr5279zTtF154AYBffvmF6upqXC6XvOtpaWkkJSVRW1vL4cOHiYiIACA3\nNzeg/MAb54WAEpehqamJqVOnsn37dh/h5A0xNvumm27i3XffpV+/fn7397pcLn755RfJpNxuNwkJ\nCQwePFi6zpxOJ263m6CgIKnZmc1mdu7cyffffy81QGEVtHZtxzIZt9tNZGQk6enpOBwOwGMB7Ny5\n0ycLqri4WPrFA6UxCQvg//7v/1i8eDFOp5O0tDQApk6dypQpU4iJiaGgoICpU6cCUFFRAUDnzp25\n/fbbAf+6QwSdXHLJJRw8eJDQ0FBuueUWACZNmkR6erq0DEaPHg1A7969mTVrFoWFhbz++usA/P3v\nf5eukTNFUFCQdA3/2v8HOHz4MA6HA51OR1JSEtB6mhZ04007DoeDhQsXMmfOHJYsWSIZv3dWqBAQ\n4m6pqiqVruXLl7Nv3z5mzJjBwIEDAVq9R6qqYrPZKC8vZ//+/fTs2RPw8ISQkBAaGhoknel0OmJi\nYvxC0yezLsFzz3ft2kVhYSFXXHEFABkZGccxeLF3mZmZXHTRRaxbtw7wuNNaY0WdCEJZnzFjBmvW\nrAE8dzA+Pp4BAwZw3333AZCdnU1QUBDz589n6dKlspxi0KBBfps+/lv434mgatCgQYOGcwrnhQUl\nLIBFixaxYsUKn/iN+PNYTWffvn288sor/O1vfyM7OxtABuKP1RhPRzNXVZXS0lLuuOMOuY4LLriA\n6dOnM2DAAGnSC9PfarX6+PCXL19Ofn6+tADHjRvnVzek+N4jR44QERFBbGwsHTt2BDwFqKWlpbjd\nbunOqqiooLy8nNTU1IBoTU6nk3nz5gEwd+5cbDYbkZGRPPXUUwBce+216PV6LBYLr732mkwUUFUV\nk8nESy+9JNfqLzgcDv76178CHhejwWDgz3/+M4899hiAT9wiJiZG/t6NN96I2Wzm9ddf5/Dhw4BH\ng09ISAh4vBPg9ddfx+FwEBcXJxM6/AHvDDPwxFBmzZrF+vXrcTqd8t1E9llCQoK0FjIzM+nYsSMO\nh4OCggIAFixYwI4dO3jkkUeYPXu2/Fxr6NztdrNv3z42bNhASEiItOqamprYvXs3paWlMtsyNDT0\nOI9CoM5Hr9cTHx/PddddJy2Qk72nTqejT58+/Pvf/wY8seILL7zQr+vT6XRERETw+9//XlpyNpuN\niy66iGHDhpGbmyvXXldXR2lpKVarVWaGhoeHt1kW3zkvoLwTCgSDAw8RCMablJREXV0dFRUVPlk7\noaGh5Ofn06lTJ+Boht2xPv7Tgd1u59lnn6WmpoYhQ4YA8MUXXxwXH1EUheDgYMLDw2UmT3R0NPX1\n9TQ0NEh3iAiy+quoVrx/VVUV9fX1xMTEkJKSAsDgwYMJCQnBZrNJZqSqKt999x1jxoyhS5cufvU9\nK4rCli1bePTRRwEPMzEajVxzzTVMnDgR8PjmnU4nixcv5vvvv5cCNigoiKysLEaMGOF3Ab5u3Top\nNPV6PVOnTmX69OlSMAkF5ljXa2hoKJMmTeK///0vBw8eBDz7HB8fH9ALLQLdv/zyCwaDgSFDhtC5\nc2e/P0fQzqJFi9i4cSNOpxO9Xi9dUImJiYwcOZIrrrhC0r4QBi6Xi379+gHw0Ucf0dzcTGFhIcXF\nxQBkZWW1am0ul4tt27ZRU1NDnz59yMnJATx0ItzWsbGxgCcueLKY7okym3/r/LyLbo/9rsTExFPO\nEhT3PD8//5SeeyYICgqiW7duZGRkMHjwYADWrl3L3r17+e677+QdUxSFjz76iNWrV9OnTx+ZONVW\n7j04DwSUoigyHlFQUCAJJS0tjb/85S+Ah/j/85//sHLlSkkkycnJBAUFkZCQILUtoRm43e5TqkY/\nFqqqsnPnTn766SdCQ0N5++23AXyyzwT0ej1BQUEEBQXJLLqlS5eyfv16OnXqJJM5DAYDbrfbL/EV\nVVUpKSkBYNOmTdjtdnJycqTQGTt2LJMnT0ZRFBn437VrF9u2bWPNmjVER0fL2EZrhYKqquzbt4+7\n7rqLxsZGwEP4EyZMYMaMGTLwrygK1dXVvPzyyzK+Ap6zeumll6TQ8AdUVeXw4cPcc889kklNnDiR\n119/ndDQ0OM+fyJFJiYmhl69erF//34AHwsjEBD7CJ4kBZPJxLRp06Sy5a/2OiLGA54Yks1mQ1VV\nwsLCpAfiD3/4A1dddRXx8fGSXsVzVVXl22+/BaC+vl6uq3///q1an7cwKSoqIioqiuuuu07GcB0O\nB2PGjCEqKkpauwaD4YQdHMT3CcF77Pp/TQDBr98Ht9tNaGjoad3fr7/+WiYupaamBkxIhYWFSUWm\nqamJb7/9ltLSUubOnQt4slVLS0sJDw/n2muvlRmHbVmbdU4LKJGN8v333wMewhfm61/+8hdGjRoF\nwLfffsvevXuJj4+XVlVmZqbMxPK+zG63G5vNJhnk6cDtdrNy5UoUReF3v/udtEx+7UCFZjlnzhwA\nPvjgA0wmEz169JCX3uVyycya1goFs9nMq6++CkBJSQnjxo1j/PjxdOnSBThaC6KqKpGRkYDHqkxI\nSGDv3r0UFxfL+hIhYE/2fieCd8bek08+SUlJiQyQZ2Vlce+99xIRESGZ4YEDB3jxxRelpi0ueo8e\nPejVq9dxxY6tgdvt5pVXXqGiooKLL74YgFmzZh3nEjrZOQQFBZGeni6z6PztfjwWNpuNP/7xj4DH\nes/NzaV///4+GX/+YCjeAso7CUmv18vza2lpwWq1Hpfxp6oq1dXVPPvss/JnOp2OcePGSXpqLRRF\nobGxUd5/caeFl8JgMPhkDHrXRIl/u1wu3G43lZWV8vcjIyOxWCwkJCScMJnqt+6k0Wg8rQxcl8vF\nyy+/LPc0Li7ulH7vTCHeqX///jz33HP84x//YMOGDYDn7lmtVjp06MCoUaP8Xl94KtCSJDRo0KBB\nQ7vEOW1BATQ2NrJq1SrAo9mYTCZycnLo27ev9M3v27eP4OBgxowZI+NNmZmZpKamYjQaZbxFxHtE\nG5LThU6nw2w2Ex4eztSpU6XG8WuxI5GOPmvWLPkuXbt2JTExUVbiZ2Vl+SXG4na7ef3112VaaXp6\nOvfffz+dOnU6zv3grXV37NiR6OhompubKSgokK6T+Pj4MyogFgkhTzzxBN9//z1Op1O6DVNSUli5\nciXr16+XNSMrV66kpqZGJkUIzbJv374cPHgQk8lEYmIi4GvVnQ6ENl1SUsLChQsJCwuTlqZ3e6xT\neU9VVSkqKpI1I2JtgYA4U1EbFhQUxPXXX09ERITf6/uE6xs8NFFUVCRri0TNTmFhIV9//TXTpk1j\n6NChgEdDb2ho4I477pBp3uCxLGfNmuW3dYqSjdraWtavX0+HDh0AZGcR75Ry0XXCYrHIdO6wsDAq\nKio4cOAABQUFMt6Sl5dHfn6+jIn+Go6942fqWt2xYwcul0uWLpyJJ+dMoNfr6d69Oy+99JJsnfXS\nSy+hqiqDBw8+a23izmkBJWIGO3fulD+LiooiJyeHlpYWNm7cCHgubnZ2NhMmTJABXYPBQG1tLUuX\nLpV+2N69e8tkBkHE4vdPBUFBQfzud7/j+++/Z9OmTbIFj3cdgxCaR44c4d133+WLL76gqqoK8BCj\nIEgR5I2JiWl11b2qqmzfvp25c+fKWMrbb7990swpQYwiTrZ27VqKioqkgL/00ktPOy7mcrlkdt7c\nuXMxm82ykBM88YKNGzfKYk/xfKPRiMlkkkwIPIH6X375hSFDhnDjjTcCHjfFmRTrCmYyb948VFVl\n4MCB0t9+uvt+6NAhioqKZH2P0WgM2MU+cuQIr7zyihQc4eHhTJo0KWCuGOGOu+GGG6isrKSsrMyn\nKLelpYWtW7dy5513Svd2eHg4NTU1lJSUyM/p9XruuOMOmbTQGnjvrdlspr6+nhUrVtCnTx/Ao3Qm\nJib69N1zuVw0NTVRXl4u72NwcDBJSUnMnTuXQ4cOSbd3Xl6ezF47HQje0dLSQnBwMCaT6Tdd4k6n\nk7vuuouYmBhuu+22k342ENDpdISFhcmek1arlZiYGB5++OE2bbnkjXNaQLlcLgoLC2WQ3e1243Q6\nKSwsJD8/XxbF9uvXj4yMDFpaWti8eTPgudzr169n//79Mv10zJgxDBs2jKSkJCwWi9TsTxUiRbRv\n37789NNP8ud9+/bFbrfzzTffyL5px2bpgeeS5ObmMmXKFFmo6o9iwoaGBu68807Ky8u56667AE+7\nmZMxX+9CwxUrVvD555/7JCSILMTTWVtjY6NsNWU2m+UlFrEN0ejTO64UGRlJWFgYzc3NWCwWn6Ls\niooKKisr5Tl36dJF/v10IJ6/du1arFYr4eHhZ3Qh3W43c+fOpbq6Wq4jUMLC7Xbzz3/+UzJA8CR0\nCMHgb+h0OhmvmDJlCn369OHTTz9l/fr1MqXe5XJhNpuxWq2yVZXdbj/OI5GWlsbf//53vzLfkJAQ\n0tPTqaurIy8vT1quUVFRuN1un7Y+drudsLAwunbt6pPtd/jwYXr16kWXLl2YMGECgCyv+C0caz0J\nr8zu3bupqKigpKSE7du3Ax5rLTg4mJ49e8qUbqPRyHvvvSeLjMW62rpX6oEDB3wU+ylTpkhhfTZw\nTgsoq9XKihUrqKurAzyMxmazsWfPHgwGg7QWhNvlyy+/pLKyEvAwbYvFgsFgkHVU/fv3l1X7Z9p/\nLiwsjP79+/Ptt9/yyiuvAEcDsU6nUxKcyWQiKioKi8UiLaxbbrmF+++/n7i4OL8F/hVF4d5772XH\njh3o9XouueQSn+/1rvnydlOINW/dupXXXnsNRVG48sorpVV4Go0i5XfbbDYp9I8cOSITMgTjCwkJ\nITk5mQ4dOsi0406dOrFq1So2btzo06FA/G5wcDBXX3014GEmZwLBTGpra3G5XD6lC6eS2i/2r6Ki\ngtmzZ+NwOGT7pUC1hCktLeU///mP7DEJ8PLLL7eJphscHMwFF1zAM888g9VqlQkhZrOZdevWsXTp\nUhloP3LkiFQ4xNruvffeVneOOBYGg4F7772XtWvXkpubK2uOQkJC5PmJ54tkIO/EF7PZTG1tLd27\nd2fo0KHSgj5VF3ZRUZEUKoD0AKxatYpVq1ZRUFAgPSVOp1O6x4Ui07VrVw4fPkz37t35f//v/8n1\nH1uTCYGzqpxOJ9OnT5fP69KlC2PHjm2ztkYngpYkoUGDBg0a2iXOaQvKaDSyc+dOaQGBR+NRVc9c\nGKEZe3dm8LZmwKONCN/6xRdffLpt5I+DXq9nypQp7Ny5k88++wzwdSn07dsX8PTYWrt2LUuWLJFV\n94899hhhYWF+1ZBqa2vZuXMniqIQGRkpNUNAzsQR+yRmL7ndbtmx4ZVXXqG2tpapU6dy//33y/Tz\nM9mfyMhIGe8rLS3F4XCQnJzMZZddBsCtt95KRkYGQUFBsnj5u+++Y//+/cTFxZGWliY7JKSlpdHS\n0sKECRN8kiTOBOJdjEYjbrebgoICaRV07NjxpOehKIpMaLnnnnuoqqpi2LBhDBo0yOe7/Qm3283M\nmTOlFfq73/0O4Izcm2eKoKAgmbQinqsoikw+2rZtG4BPopD43OWXX/6r33umNX+i/+DIkSMxGAzS\nQhOJMyc6Q6fTKTvUL1q0CIDLLruMtLS04+qgfgtRUVGyBlNRFPneF110EaNHj6ampkZaUDt37pRh\nieTkZAAqKyuJiYnhiiuuYPjw4T5jefxVpH8yqKrKkiVL2LBhg0zwycvLIysr66wkRwic0wLKYDDQ\nu3dvOe7A5XKh1+sxGo1YLBYfn7P4vBBUInA5fPhwHnzwQcAjNFpbbyTqMGbMmMEzzzwDINckiBc8\n2WybNm0iJSWFSZMmAUer7v2JsLAwunXrRn19PUOGDPHphiCy9bxNeIfDQVlZmaxZKS0tZfz48Tzw\nwAOtyuQRAVjRiHLixIlERkbSv39/n5orURsmiq8///xzdDod2dnZ3HrrrVx11VWAb31Ja/dMuIL7\n9+9Pfn4+JSUl3H///YDHbSZGNhzbCcThcLB+/XoeeeQRwFMflJiYyG233ebXAuJj0dDQwKefford\nbicqKuqsFFB6P8/bXSwSegSDDgkJQVEUXC6XXOfJlLDW3j3vdmXiZ8fSiFBgf/jhB9nNOzMzkyef\nfJKMjIwzEpCpqak+dXJCwPTp00fux7HvrCiKVMQWLVpEZmYmeXl5REZGHpdQEeizramp4cUXX8Th\ncDB27FjAM9ssUDHNU8U5LaD0ej1/+tOfZOJBfX297AZhs9l84ggiriSy5IYNG8YDDzxA9+7dZTGl\nP/33er3+hExKEFpxcTHr168nOzubCy64AAiMtm0ymZg+fTrPPPMMJpNJxusiIiIkYxYWqMViYfPm\nzcybN0+26pkwYQJ33323X/pv6fV6aVn079//hBluquoZznf33XcDsGHDBqKioujRowfDhw+X5+fP\nvRLn/vDDD7No0SL2798vNerDhw8zfvx4RowYIZ9dWFjI9u3bWb9+PTt27JD7l56ezpNPPsnVV18d\n0FiQw+GQiR0RERFy9lRbB9SPhejMv2jRIsl4k5OTaWpqwmq1Sno7WW/C1tCYKBq2Wq0npCvveVCf\nffaZT1brQw89RHZ2dqvOzVuYCAHtbYkduyadTkdzczPg4V3Z2dl06dIloJMDjoVQ4r/66iuqq6uJ\ni4vjD3/4AwDdu3f3e6zwdHHOC6i+ffvKQWivvvqqTB0VEzPF53Jycpg8ebIMqGdlZUkG2Zaap7Dm\nXn75ZVwuF5MnTw4oERgMBnJycnj77bdZtmyZnLQaGxtL7969MZvNFBYWArBx40YOHTqEw+Hgpptu\nApB1Nf7YI++Le6LsNpH9NH/+fJlt6Xa7ycnJ4cEHHyQlJSWgU1lTUlJYtmwZI0aMkC2htm7dyvbt\n26WmDUfdW06nE6PRyJVXXgl4art69uwZ8EQF7+8PCgqS/z7bAkpVVQ4dOsS+ffsk4xeD+JxOp3Rn\nnWx0SGthMBgICQmRe6EoCk6nk8bGRjkw8Msvv2T79u106tRJnmt6erpf+cCJhJE3xMwuUSITGhpK\ndnZ2myYkKIrCjh07AE+bNYfDwfjx42WKvslkOmvp5QLntIACD0EKt8+2bdtYv349paWl6PV6GVtJ\nSEjghhtu4Nprr/W5JGfDtyoElMViITMzkx49egR8pLrBYCAuLo6rrrpK+sEPHTrEgQMHOHDggPxc\nSEgIF198sU8MxTsLKtAQrpfNmzfLZ8bFxfHQQw/JBp+BhE6nIyUlhe3bt/PNN98A8Oabb7Jv3z4s\nFotUJMLDw4mOjqZHjx5cfvnl0kXbVnsVHBzsU5QsPACBpqPfgnAXd+vWzcc11NTURFNTk/QoiEzJ\nQOyVWIN38f2RI0dYvHixTPMODQ3lueeeY+DAgVKQBurcTvS9quqZti2sJ/Bkq7bV5GqxhtraWt56\n6y0AVq9ejdFoZMCAAZLOz2bsSUDL4tOgQYMGDe0S57wFpdPppGb21FNPkZ+fz6JFi6itrZV+8KFD\nhzJ27FiSkpL8Pj33dKCqqszeSUxMpEePHifsdB4ICM1SNMtNSUnB4XDgcDh8WtBERka2KouxtXC5\nXERERMgxAIMHD2bo0KF+GXl/KhD7JKyiiRMnykJTEWsSbjXxX1vTUkhICEOGDGHVqlUyMaA9QFVV\n4uLiGDp0qCxEP3z4MBs2bPBZo7c1FQh4J/643W6Sk5OZNGmSdMXGx8f7dHZoa6iqitVqxWazkZmZ\nCXg8BWFhYQHtPOINRVEoLi6WrnTRZGD06NE+tYZtHQI5Fue8gIKjwsZkMjFgwAB69uyJ3W6XGx0W\nFiZb7J/NzXa5XHJoW0hICHl5eQHJ3DsVeKcJiyw6gbZej3fsxOl00qFDB9lL7aKLLvJ76v3pwDtu\n1l4EgclkYtiwYaxZswar1SpjK2fSksefEMri2LFjZZp+fX09W7dupaioSNJZRUUFCQkJPmMvAgW9\nXk9oaKjPuJSzRUuCzh0OB1u3biUhIUHSeVu698DDiw4ePCi7gOj1evr06ePjRm8PLj7d2Q6s/grO\neFHeXREEzrYWAEfjK7t27QI8saiePXsSGhp6VtrYt0coikJzczMNDQ2ytig7O1sK8bN9hhpODtG/\nUoyIAQ8jfPHFF2lsbJQ/u+mmm7jwwgsxGAxnPQjflhAlJg6Hg5aWFiIjI6Wl15bWnOBFv/zyi2x9\ndtFFF/Hoo4/SqVMnHwvKTzjji3veCaj2DO9UV+/6DI3xHkWgCxI1BB7H8hTRB0+kNJ/qdNnzDccq\nzWeT1oUiMX/+fADGjRtHVFRUoM7ljF/yf49KNGjQoEHDOQHNgtKgQYMGDYGEZkFp0KBBg4bzC5qA\n0qBBgwYN7RKagNKgQYMGDe0SmoDSoEGDBg3tEpqA0qBBgwYN7RLnRYWod+di8afb7ZZ1F2azmZCQ\nECIiIrSiWA0aAghVVbFYLMyePRuAp59+GoPBwJIlS+jevTvQPjoUaDg3cM5za1VVpYByOp00NTXx\nyy+/sGPHDjlaorq6msGDB/Piiy8SFxd3NperQcN5DYfDwTvvvMPf//53ABobG9Hr9Xz00Uc8//zz\ngH/nrp0pxCRpISwVRZF98P4Xi4jbK855AeUN0QusV69ebN68mSNHjgCefmD79u2TXRzaC7SuCb8N\n0ZZFzPcSPdXacqjb6UI0l4W2Wae3giYa2oqRCW25T6qqsmfPHt58802fURLBwcEkJiae9XlV4kya\nm5v5/PPP2bZtm5zGHRcXx7333kvHjh2Ji4uTwylF41l/7KH3899++20KCgrk+J8xY8YwcuRITCZT\nm9CLENBCGNtsNg4cOIBer5c0FBsbS0hICAaDgZiYmLPifTrnBZR3qyCTyYTBYCA1NZVBgwbJRojr\n16+nvr6egwcPEh8fD5y92Tne/bgAnxHzgSJM0SNNURTZZXrv3r3s2LGDrVu3UllZCUBLSwuDBg3i\nT3/6E0lJSQBnrcGuYGY2m42PPvqI+fPn0717d6677joAhgwZctanfQqIHnPr169n9uzZhIeHy/2b\nNm2aZHaBghAGCxcuZP/+/SQnJ9OzZ0/A0w2+rYbgWa1WnnvuOWprayXNREVFMXToUAYOHHhWFQpV\nVfnll18AuPXWW6mtrSUuLo7Y2FgAunTpwg8//MCYMWNobm6WXcbFyPrWWH2KomCz2VixYgUAs2bN\nYmhyiHoAACAASURBVPXq1T6Tf+fMmcPQoUOZNWsWHTt2DDh/En0vt2zZAkBdXR12u50BAwZgsVgA\nz7DOqKgoOXXcezpwW0GzZTVo0KBBQ7vEOW9BHQu9Xk94eDiXXXYZMTExgEezq66u5qeffiI7Oxvw\naHZtCTFFU4xGWLRoERaLhZycHEaNGgV45tQEyiXjcrk4fPgw7733HgDz58/HYrHQ3Nwsp/wqisIP\nP/zAzJkzeeqppwCPtnk2RoIIq2TZsmXMmDGD6upqioqK5EiJoUOHtul6TgRFUSgrK+PGG28EPBqn\nXq/n7bff5pprrgECP6JDURQ+++wzAH788UfCwsJoaGiQtO9yuQJuQQnX+cKFC1m/fj0Gg4EBAwYA\ncPPNN5OcnOyTtHQ2YlBlZWVMnToVgNraWqKjo5k6dSojR44EICIiAvCcl9FolOfW2r1zu90UFRVx\n3333ydlLdrudoKAg4uPj5QgSp9PJihUrGDx4MFOmTGHGjBlyPf6GiLNFRETI8e5lZWUEBQURGhoq\nPSqHDx9m+fLlZGVlMW3aNJ+RJW2F805AwdFZR926dQOgd+/e1NXVYTabKS8vBzwE2VZuPpHZ9N57\n70kBcejQIZngIYh04MCBjBgxgj/+8Y9ER0cD/rnMiqJQUVHBrFmzKCkpATyEb7PZCA4Olu5G4X6s\nr6+XAe1evXoxfPjwNnMTgWe/du/eDcCjjz5KVVUVbrcbt9tN//79gbM3cBKOumxmzJjBjBkz5P7p\n9Xr69evH1VdfHfBR4gIOh4PCwkLAw0xTU1OJi4uTbr+qqirS09MDSuvCbbxgwQJaWlro3r27pJ/e\nvXtjsVh83H5tDZfLxd13301VVRXgUU4ff/xxJk+eTFhYGOARGsXFxeTn53PBBRdIoet2u8/oDgoF\n6/PPP+fBBx/EYrFImhgwYACXXnopN998s/zu2tpavvjiC+bMmcOcOXPkXXzppZcICQnx+96JYZtC\nAIrBqYqiSDrfvXs3dXV1cm7c2cB5KaAEhJWUl5fHxo0biYuLOyvam9Vq5fHHH+fTTz+V02uFFqMo\nipx9tHTpUpYvX84333zDE088AcDFF1/cqsCpCIhWVVWRlZUlY3BpaWlYrVYKCwulMGhqapLEKS5u\nUFAQTqezzYLtiqJQVFQkp5+WlZWhqipGo5HOnTtLC8qfAVtvwSNimuJn3pdW+OZ37NjB/fffT2Fh\noc9gzClTpvD++++36WUuKipi+fLlAKSmpnLLLbfQ1NTE9u3bAejTpw+pqalAYOKuiqKwbNkyANat\nW4der+fCCy+kd+/ewFGL5GyWeMyePZulS5dKpj9u3DhuvvlmTCaTFEQOh4Pdu3dTVlbmM/jxTMbh\nOJ1OmWb/8MMPExwczE033cRDDz0EQGZm5nF7kZCQwPTp0/njH//Ia6+9xk8//QRAjx49uPXWW/1O\nU+KdvONKgleIPampqcHtdpORkREQIXkqOG8FlE6nk5tvNBrZvn07AwcOlBMs22KzhUvjtddeY968\neaiqKok/Ly+PTp06UVdXJzXggoICmpqa2Lt3L//85z8Bj2Y1fvx4IiMjz4jBiASJ9PR09Hq9tMya\nm5vJz8+noKAAm80GeBhYUFAQ6enpPPzwwwDk5uZiMBh8GLY/4V0m4Ha7OXLkCNOmTZParnheREQE\nkydPlq4Yf67DO7tMJJMIZibW19LSwoIFCwCYOXMmxcXFMqtwzpw5AFx99dVtPnjuiy++oK6uDoCu\nXbvSpUsX5syZQ21tLeBROiBw9G42m6WLsaGhgdjYWK6//vrjJtgGBwefFQbndDr5xz/+gcvlIiMj\nA/BY5aGhobhcLrl3CxYsYOfOnWRlZZGRkSEFwukqtC6Xi6effpo33ngD8CRZ3HLLLTz33HO/+V0G\ng4GMjAweeeQRDh48CMCHH37I2LFjycjICChteStlwiLetWsXsbGxXHfddWctqUxLktCgQYMGDe0S\n560FBUc147179xIVFYXFYmkzN4Pb7Zaus9mzZ2O32xk8eDAzZ84EID09HZ1Oh9vtpqGhAYDNmzfz\n3//+l9WrV7Nt2zYAwsPDsdlsTJo0icjIyDPSQoUFEBQUJK2ltWvX8sknn7Bt2zZp6ZlMJgYMGMDz\nzz9Ply5dAAgNDZXak6Iofk2J967HAI87b/bs2bI8AI5qsHFxcYwZMyYgmpx4lxPVh9jtdsxmMz/8\n8AMLFy4EPIXfJpOJ5ORkXnzxRZkQ0dYWggjAC22/V69ebN++ndLSUmkpCws4EGtTVZV9+/axdu1a\nwEMfnTt3Jjc314dOhBV6NiyoTZs2ceDAAQwGA1dffTXgcW+73W5KS0t55513ANi2bRvdu3fn8ssv\nJyEh4YxCAaqqsmbNGt58800Zlxz4/7H33vFRVfn//3NKMumNdBIgoSV0pCNKExQsFGFt61oWXBW/\nKq6rrq5i17WguJZ1EV1XQSmCVAUEkUCQDgk9gRRCSG+TTJ97f3/M7xwniI3MBORzX4+HDzSGuWfO\nPeddX+/3u39/Zs2a9as/T3ibIldeW1vLqVOn/E499w5ti2efPHmSwYMH065du/OWP7xoFZSiKNJ9\nP3jw4I/YTP7ccFVVaWhoYMaMGQCUlpaSmJjItGnTSE1NBX6oL9Lr9bK7xdChQ4mPj6e+vp59+/YB\nHiFUXV2N0+k8p0uuqirFxcW8/fbbHD9+XBYmVlRU0NjYKENZ4AmH2O12srKyZA6qtraWDh06/CgG\n7SuBIwpxwRMuEoQIIXQtFgsBAQG0b9+ejh07+iW0J/ZAjCV3u90yzLFjxw62bNlCbm6uNDj0ej2X\nXHIJN954I9dcc815u7z19fVUVVVJxl5ycjI7d+4kKipKhmjbtGnjt/W5XC4+/PBDuVfh4eHcdNNN\nBAQEyL0VIWaHwyGFdGsUo4o8yqxZs7Db7URGRjJhwgTAY/RUVFTw3nvvsWPHDsCj3G+55Ra6du3a\nIkJQbm5uM0WSmZn5qz9PVVVsNhvz58+XIe7U1FSMRmOrhdiamppkvZbdbufGG2/0OxP153BRKihV\nVamvr+fEiROA50II6qR40f606Ox2Oy+88IKklIeHh3PzzTczaNAgya4SgjA0NFQeYJPJRExMTLPK\n9eTkZEaNGkVoaOg5HVKbzcbatWtZvXo1FRUVkqThnfsRcLlc7N69m5ycHN58803Ac0HGjx/PzJkz\nm63VF9DpdBiNRkJDQwFP8viKK64AkIIjLy8PRVFk4aQvc2HCI/T+TLPZTHl5OZ9//jkAmzZtoqGh\nAbvdLp8ZFBREeXk5BQUFFBYWSrZoa+ef9u3bR8eOHeVzKysrKS8vZ9q0abRt21Z+J3+hurqaLVu2\nSGXQsWNHevToQUNDg/TUKysryc7O5vvvv5ee+j/+8Q+6devmV8KSIB7l5+ej0+kYMWIEaWlpABw4\ncIDZs2ezY8cO+bMrr7ySPn36tEgY63Q6unbtSnR0tDS6CgoKcDqdZ/2u3vfP4XDQ2NjI6tWr+fjj\nj+XPp06dSqdOnVrlbKmqSl5eHuvWrQM8nSQyMjLOa4H1RaegFEWhvLycpUuXStZVeXk51dXV1NfX\n+73dkcvlYt68eSxdulQeyqlTp3LzzTdjNBolu+r06dOEh4cTHx9PRkYG4DkgNTU11NbWSqtYCObf\nepm9D7/b7cZisfzouwu33vsACgq1qI2qrq7myJEjbNmyhSeffJLLLrvsnNbzU/Ams4SHh3PFFVfQ\nq1cv3nnnHcBzwR0OB1u3bqWwsNCnDUcNBgNut1t+lsPhID8/n2XLlvH1118DnsR/UFAQ7dq1k6xQ\nu91ObW0t8+fPZ8mSJfz3v/8F4LLLLms1JeVwOCgtLSUsLEyGZMrKyhg+fDgDBw5slU4Ey5Yto6Ki\nQnqgbrebJUuWUFlZKftgNjU10dTUhMvlkmfyq6++4oknnuChhx7yS/mCqqoUFhYCHgMtNjaWdu3a\nyVDkiy++SEFBATExMdxwww0AjB071iee3YABAxgyZIh8J6GhoaxZs0bWW4Gna0NjYyMBAQEcP34c\ngMWLF7Nz504sFgs2m02SgaKiolqtxMPhcPD+++9TWVkJePZERFLOFzSShAYNGjRouCBxUXhQwjJz\nOBysXr2axx57DJfLJcNZVquVkJAQYmNjZQK+bdu2BAQENOuF11K43W62bt3KRx99RH19vfQ2rrzy\nSkl2EM9KTEzE6XTStm3bZvmW7du3U1paKvNSgwYNIjMz8zevUViCISEh/PGPfwQgOztbhh11Oh0D\nBgwgMzNT1skcP36c7du3c+DAAWlF2e127HY7O3fu5N1336VHjx4Afqkp0+v1REREoNPpZHeG7du3\nc+jQIerr61m1apXsBOKrd+ZtMQcEBBAfH4/FYpEkkdjYWCZPnkxmZqYMRTocDtasWcNrr71GRUUF\nzz77LACrV6/2e989+CEUY7PZUFVV5iuER9AaRKDGxkY+/vhjamtrJcmluLiYY8eOYTab5c/O5pFY\nLBaefvppCgsLpafsS49PVVW++eYbwBOObdOmDQcPHpR0+Pr6eoKDg7niiisYN24cgM9o8OHh4bz7\n7ruyUXV+fr7syCDq1crLyykvL6ekpISysjLAE/IPCgoiJiYGo9EovaZFixZhNpuZNm2a33NBdXV1\n7Ny5U0YK7r777vPe2f13r6AURZHhqM2bNzNnzhzq6+uxWCxSQRkMBtkcUcSmQ0JC0Ol0REdHy5qN\nlr4Mi8XCY489xvHjxwkPD2fEiBGAh/wQFBSEy+Vq1qxWr9djMBikgq2uriYnJweLxSJzMV27dsVk\nMp2zMtDpdMTGxvLAAw9w1113yTxAQEAAQUFBzb6zYPE4nU6+//57AD7//HNWrVpFTU0NGzZs4Lnn\nngPglVde8bmCEuHGkJAQWbOSlJTEyZMnadu2LRkZGT6/MKJYGjx1KImJiUyfPp34+Hj5/DMFvtvt\n5vrrr5dC7+TJk4Dn/beGgnI4HERHR1NeXs7atWtlOGjq1KmkpKS0Sjfsffv2cejQIRwOh3y+0WiU\nRd1CwEZERNC7d2/CwsKk0bN3714sFgtLly5l1qxZACQkJPhs3dXV1ZJxabfbOXXqFIcPH5Z5IVFv\n1L9/f7l2QeZo6ZnW6/VERkZKIZ+eno7VaqWkpISdO3cCHiWWn59PY2OjPFtt27blT3/6E927dycr\nK0uSpPLz85k7dy7p6elceeWVgH/yiuKdnjx5kuHDhwPIRrrnE79bBSUOVHV1NdnZ2YCnb5uqqnTu\n3Jnq6mqqqqoAT6L/sssu4/rrr5fJY7vdjsvlIioqqsVjAMTf//zzzzl48CBOp5OOHTty6aWXAsgi\n27NVgyuKQn19PQCffvop3377LYqicPPNNwOeGHRLL4234PcmBJztoAua67BhwwBPN+zJkyczdepU\nzGYzS5YsAeDhhx+WVHlfQdC8dTqdvOB/+tOfsFgsJCcnExER4ZeRDd6U6IiICC655JKfVYQGg4GY\nmBjuvPNOli5dSlNTE/BDoaq/FIR3rsflcrFv3z4qKiqkkJ04cWKr5CtcLhfvv/8+drudsLAw2Tk9\nLS0Nq9WK1Wpl/PjxAEyePFmSBgSD9KabbqKwsJCmpiZWrFgBwB133OGTtbtcLl544QUOHz4MeJS5\noii4XC55j0SH7tTU1GYF2b56b2ebsNChQwduv/12AA4dOsS1114rZ1CBh0WYmJiIwWDgqquuoqio\nCPB0JykoKOAf//iH7Nnpr7ydKIcRiikwMLAZmUp8p9YkTfxuFZTb7aahoYHdu3dLWmRJSQmpqalE\nREQQGBgoiQaTJk2iY8eOzdhxDocDi8XSbJxESxli3333HVarlYCAAHr06NGM3fVTn6koikzIz58/\nn/r6eiZOnCgvvS+9FFVVZejll2pjhIA2mUzEx8djMpkwm82SUlxTUyMp876CaLPiPTQuLi4Ok8lE\ndnY2vXr1kkrfH/gtbW1ESyaDwSB7KfoT3u/OZrNRU1NDdXU1er2elJQUwL+Ucm80NDSwb98+VFUl\nMjJStqXKzMyktraWtLQ0GQoWkYPq6mqWLVsGeBh2Op2O4OBg6VX5wvBQVZWCggKWL18uoyrC4DKZ\nTDLhn5CQgKqqBAYGtsp8MXGuhTwaMWKE7IMnPKgzz156ejoAPXr04PDhw5SWlsrv5A8F5XQ62bp1\nKwaDQXa7CQgIaNb6SCAwMLDVQn+/OwXlnW8Scd28vDwAunTpQnFxMfn5+YSGhkovRHQJ9/774LGk\nXC6XfAEixGUwGM7pBbjdbhk6Gzhw4C8yYEQT17lz5wKe2qROnTrx+OOP+zzeLGosBJMpMzOTxMTE\nX/yeqqqyevVq2adPXGhfdjj3bnUENGs1tGbNGrKzs3E4HM16BZ5PuFwu9u/fzwcffICqqjKU68sw\n1dkgPluv12M0GjGbzc2Elb/7TIq9r6yspK6uTobKhDHV0NBAQUEB9fX1LF26FPDkqqqqqigpKZHM\nNvBEFfr06SND2b7ImzU0NDBnzhzcbrf0KsPDw2W0RexVWFgYY8aMYdCgQfKenUvPvV8Lu91OeXm5\nVFBiCODPGa7iXoq+fd5Nnf0Bs9lMU1MTOp1OGqHetZrecvJc5eO5QGPxadCgQYOGCxK/Ow/KG0VF\nRVgsFmlZ5OXlUVRUhNVqJTo6WrKuRCeGkJAQmcS22WwoiiILZsHD9ktKSiIiIuKcLIRbb72VrKws\nWXzq3aXhTKiqSl1dHY888gh79+4FPJ7eiy++SPv27X1uodjtdvbv389rr70GQOfOnXnyySdp06bN\nj54lLE7wtDt5/fXXZaGqYNGJibG+gLDMy8vL0ev1hIaGSoJLVlYWZrMZg8Egw0G+hOhgLvBTFq13\nx4vvv/+e2bNnU1BQQEJCAn/6058A/Foz4p0jEfnMwYMH09TUJEPJ/u6iLs6EmAbrcrk4fPiwJDoY\nDAZKS0uxWq0yL6coCkajkZiYGDp16gR4ku8DBgxgwoQJsq6tpeddURR2795Nbm4uoaGhMuzau3dv\nyQIV8uDpp5+WI2Rao6NFdnY2GzZsYMqUKXJNvzSpWpCZjh8/TlhYGJ07d/brWhsaGggMDCQoKEjO\niBJELvHvovFwQEBAq003+N0pKLEpRqORYcOG0aZNG0mTzs3NZcyYMXTs2JHIyEgpTIXgqK6ulh0l\nysvLSUtLIycnRzLWFEXh9ttvZ9iwYb8paSp+b+TIkQwfPpy9e/fy6aef0r59e8BTvCdaBYk8Qk5O\nDtOmTZPhSPCMgr7iiiv8EqqxWq3MmjVLjniuq6sjLy+PiIiIZmEiUQD6/PPPA7B27VrZMiowMFAm\nakUIxRcQe6KqKrt37yY1NVWO587Pz5dCLi4uzieKW7xboYi9c5BnExyiM8lLL70EeOj6jY2NpKen\nc8899zBgwADA/+2zxD4ZDAaCg4MxmUykpqbKYlN/h/jE94uPj5fM2JqaGtlLUqwTflA4gYGBDB06\nlKeeekreR1Gi8EtC+rdA9Es8ffo0iqIwaNAgwGOY1dfXExUVxeOPPw54ckCtFaLS6XQsX76cFStW\nSAH//PPP/6QxIULxolWVyK9nZGRIOeEPREVF0bdvXyorK2Vu2XsER319vXzPsbGxflvHmfjdKSgB\ng8FAZmYmXbt2lYe8qamJoKAggoKCml1W4SmJdiLgibmuXLmSlStXyvj3pEmTMJlMOByOc5p/EhQU\nxBtvvMFbb73Fhg0bePDBBwG45JJLGDZsGEajUdJHv/zyS+rr68nMzJS1IP369fObkAkMDKSkpER2\n18jNzeUvf/kLI0eOlN5QcXEx27Zto7CwULaqEUI7KCiIe++9l0cffRTw7TwmISwcDgfr16+noqJC\nNsutqalBVVUSEhK44447fCJYxHtVFAW3293s+SLmLrxqp9NJTk4Of/vb32R3gtDQUCZPnsz9999/\nVhq6P6CqqmyTVVZWRlZWFiUlJQwcOFAOcfQ3xL4NHz6cRx55hDfffJPKykp5ZoODgwkODqZDhw6y\nge5tt91GdHQ0BoPhF++TdzPiXwuhEA8ePMjChQupr69HVVUp2OPi4hg5ciT33Xcfffr0AVq3JZUg\ng9hsNlatWiXX/MILLxAVFdWsEXNZWRnPPPMM69evl0ZhaGgoU6ZM4YknnvArQ9NkMpGUlERtba00\nDpOSkiQ5ymKxSDLOr3mXvoLuQkg6nwU+X5T4nt5MqJKSEkpLS+X/S09PJyEhQdZInQsURaGqqor8\n/Hw5E6a4uBiXy9VsrpKqqlx33XXceeedckaVPwWdw+Hgj3/8I8uXLwd+2AdofmHFcD7x/cPDw0lL\nS+Oaa67hiSee8Eudj9iT+vp6HnroIbZs2SILGK1WK3q9nttuu4133nnH55fUO/lbWVlJdXU1+/bt\nkzUrp0+fJi8vj+LiYkl9nzZtGrfddhspKSmtNgDT7XbL3pJz5sxh165dpKam8sorr8h2WK1J/3W5\nXPIfcW5F0fv56N22efNmbr31Vurr61EURZaT3HjjjTzwwANERkael3Wpqkp+fj4TJkyQ06zBQ9CK\njY2VbMddu3ZRUlIiyRyiUPyPf/wjM2bMIDQ01G/1T+AJ8T3yyCPs2LFDNgnIyMigT58+9O3bl27d\nusm7fw4K/pwXrpEkNGjQoEHDBYn/Mx7UWR/ixymxLpdLJtVLS0vZtWsXJpNJJm8zMjJISEholUSt\nWNPWrVuZPn064Em+ut3uZrRt0dlbp9PJxPu0adO4/vrrW2UmjKhjee6552RbGKvVSteuXfnss89I\nSkry+Rq8C6XLysrIy8sjKytLenCnTp3C7XaTnJzM3//+dwDZibs1LXJFUdi1axcAM2bMoKKigvvv\nv5//9//+33kdh3ChoLGxkTfffJN9+/YRFhYmw+vdu3dvtWarPwWRw1ywYAHgmQ+Xn5/fbD6duHfp\n6elcffXVsmvEpZde6lcvXdx/u93O/PnzWbdunTxPRqORGTNm0L17d0wmU0tCo+d8Uf5PK6jWggib\neeN8hEIEcxDgnXfeYfPmzc16qQUFBdGzZ09Gjx7NNddcA9CicOe5wul0ynBITU0NnTt3JiYmxudd\nK3Q6HQ6HQ35/u93OoUOHMJvN0riIiooiLS1NFgxD6w8mFOsVdUQLFy7EaDQyffr0VpmtpME38J4/\nZrVa5RgX8OSAwsLCCA4ObjWG3NlwpqzykXLUFJQGDecCt9stCSEGg4GAgIBm/RHh/CgkDRouImgK\n6kKGv0KJGjRo0PA7gEaS0KBBgwYNFxd+t3VQvydonpMGDRo0/HZoHpQGDRo0aLggoSkoDRo0aNBw\nQUJTUBo0aNCg4YKEpqA0aNCgQcMFCU1BadCgQYOGCxKagtKgQYMGDRckNJq5hmYd1l0uFxaLhaqq\nKgA2btxIWVkZY8eOpbGxkUsvvRRAjn7XoOH3ALfbLYcAKoqCy+UiNDRU9pe7UEpBzmw1dL66w18o\nuOgUlKqqNDY28tRTT5GdnQ14XnpycjLPPPMMvXr1Alp3JsyFCkVRcDgcstWP0+nk9OnTFBQUcPjw\nYcAzZ8dqtfLdd9+RmZkpZw+1sHmkBg0txq8ZKiom7T722GMcP34c8Nz9yy67jHvuuYeMjAyAVh3H\nIQZlet8fYRy63W50Op1ci9PpJDg42C+K6rcMZT1fuOgUVGNjIzNnzmTp0qVyOKGiKOzfv5+dO3cy\nb948AMaMGdMqg+bgh35vDQ0NcrpvSEjIeWkKKawzp9NJdXU1W7ZsIT8/H/B0Lw4LCyM6OpqoqCjA\nM+gwNzdXDqYrLi4GPPNsvC/SuUIMkwTPlN+GhgYiIyPl9FChCM+nMrwQW1UpikJtbS1z585l/fr1\nAFRUVBAXF8ef//xnpk6dCuCXTucul4u6ujpsNhtWq5Xo6GgAObPIu9O797Ri0RjVarUSEhJCUFBQ\ni97rz70PMZ6+sLCQKVOmcPr06WbvccWKFWzevJn09HQAnnvuOQYOHOhXmeA9k87tdsupxIAcnKnT\n6WhqapJNWl0ul8+NQUVRyMrK4tVXX+XQoUOAZ1jqBx98QGRkpM+e4wtoJrAGDRo0aLggcVF5UKqq\nUlZWxvbt23E6nXIOjLA+qqqquPnmmwH4+OOPufbaa/1mFauqSlNTEwArVqzgk08+4fDhw1itVrmm\nu+++m5kzZ8oZUa0xb0lYsZs2bWL16tXNRnZ36NCBMWPGkJiYKPeuW7du1NXV4XQ6cbvdJCUlAb4Z\n+6woCtu3b+exxx4D4MCBA4DH6hfTO7t3787VV1/NyJEjadu2rVyX0WiU6/bnpFGn00lTUxOHDx+W\n1nVcXJwc9e5t2frCo/y1sNvtrFq1ivXr18vpq8OGDWPlypXMmzdPjkvxpQclrH273U52djYbN25k\n69atMiogPN3w8HAqKysBj2cQHh5OVVWVHGMeFhZGjx49ePvtt2nTpo38u76CqqpUVFQAcP/991NZ\nWYmqqvLsBAcHExsbS0NDA9u3bwfg6quv5tFHH+XRRx/123kS+6coCna7HafTKce6hIWFydEper1e\nRhX8Me5Gr9fTqVMnQkND5aiZo0ePsnfvXoYNG9ZqkaVfgwtnJT6C2WwmKCiIjIwMLrvsMsATzisp\nKWHWrFlyHtLy5csZP368316G3W4nJycHgLfeeovS0lICAgLkhYyIiKCgoID58+dLpelv99rtdlNU\nVATAggUL2LdvH1FRUUyaNAnwXNJ27doREBAgBXTv3r2JjIzEbrcTHx9PWFgY4Bul4Ha7+f7772WI\nsbGxkcDAQBRFkZ9fVFTE1q1byc3NxWq10q5dOwBGjBhB//79MZlMUlF5z65paVhO/P2CggKeeeYZ\ntm/fTnx8PABTpkyRIbSqqiqZw+vYsSPx8fF+V1Jut5vVq1eTn5/PggULiI2Nlf/v1ltv5YUXXqCh\noQHw7ZkSe9LU1MSSJUv46quvsNvtzQQveJS6d6JfVdVm+Q6h2Gtra+V4cV/C4XAwf/58APbsQhTK\ntwAAIABJREFU2YPb7SY0NJSrrroKgAceeIDg4GDmz5/PokWLAM9Q0VdeeYUJEyaQmZnp8zXBD/tj\ns9mor6/HbrdLQ0yE+4QiFYakv8LbSUlJPPLII9x9993y+WLemTfOfHfQuqHui05B6fV6kpOTiYyM\nZObMmQDExsZitVrZvHkza9asAaC+vh632+03BaWqKtXV1YDnwnTq1IlevXpJb6mxsZHa2lo+/fRT\nqTQffvhhv03/VFUVh8Mh8xVbtmyhqamJbt26ccMNNwAQExPzowm/DoeDpqYmoqKiGD9+vE/Ze3q9\nnjFjxsg4eGFhIT179iQ5OVmSMZxOJ42Njaxbt44NGzZIJeRwOOjQoQNt2rSRl/xMb+bn9uKX/r+w\n9p9++mnWr19PSEiIzLWUlJSwadMmHA4HX3zxBUePHgXguuuu47XXXvO7BVpeXs6HH37I888/T1xc\nXLPv3b59e0aNGiVZmCkpKT4TKOI5breb3NxcmpqamnkmJpMJt9st/4HmhoJYR2BgIJMnTyYlJcXn\nwtftdnPy5Ek+//xzwJPXDAwM5Prrr+fpp58GPB6wwWDg2Wef5eqrrwbgmmuuwWw2M3PmTNasWeNz\nj87hcGA2mwE4dOgQRUVFhISE0LZtW8CjIIxGIwEBAej1eqm4RaTCH15UamqqzDXb7XaSk5N/9L1V\nVZXGjvDGg4ODW01JXXQKKiYmhvj4eK677jopUAIDA9HpdAwZMoTc3FzAY0EIi8YfCAgIIDk5GfB4\ncMOHDyc1NVVa27t372bHjh3U1NSwb98+AKqrq4mPj/eLxSSmx3733XeAx/KPiIjgr3/9q/QMvD0Q\nsTenTp3C4XAwcuRIJk6c6NODaTAY6Nq1K6+++irgUUYGg4GQkJBmimjFihXs2LGDqqoquaejRo0i\nKSkJg8HQbN2+8uwOHjwIeL5/hw4duPzyyxkyZAgAXbp0wWQy0dDQwNKlS6UyKCsr8+uZEkJ/9uzZ\nqKpKjx49fnRWdDodvXv3lsLEH0wtl8tFSEgIer2egIAAUlNTAUhISMBoNHLkyBHq6+sBj+AzGAwE\nBwfTo0cPAP785z8zYcIEnws6EeKfPn26jF4YDAbGjh3LU089Jc+O2DOj0cigQYMAj0e+fv169u3b\nR0VFBYmJiT5bl6IolJSUsHLlSgAqKyuJjY0lMzOTwsJCwGP09O7dm4SEBDlVF/zrrej1ehnia2ho\nkMpKQDCi9+zZQ1lZGSkpKQAMHTq01cKAGklCgwYNGjRckLioPCiRYIyKiiIsLKyZlnc4HBQXF0uv\nKi4uDp1O57daAIPBQLdu3QCYOXOmTICK2PK2bdsICAggKSlJutAffPAB9913HxEREX7xovR6vUzK\nqqpKSkoK7dq1a0Y2ECEZsc7S0lIGDRrEHXfcQVhYmM/3KiAgQOZJvMNBwhMpLy9n9uzZ5OXlERoa\nyieffAJ4aLFnhiN/LX7p7wgvBDz5w6ioqGbEEfE7drtd5hQBoqOj/WpZisR5TU0NM2bMOGs4WIS4\nT5w4AdAsF9RSiH2LiYkhMTGRI0eOoNPppBUuqNN9+vSRBKHQ0FC6d+9OZmYmY8eOBTx5sXN9dz8H\np9PJSy+9RE5Ojtyb2NhYZs6cSVJS0lmLckV4eNasWWzbto36+noOHz7sUw9KVVWOHTvGhg0bAEhN\nTaVHjx5YLBYKCgoAjwfVrVs3GT1ojRBaQEAApaWlco2iBssbDoeDkydP8vXXX9OpUyfAc/day4O6\nqBSUKHYLCwvDbrfLi+NyucjPz+fkyZNyY4WwFuE/X0On08nD7534F6GXXr16kZycjNFoJCsrC4BF\nixbRsWNHJk+efNaEZUsRGBgolYGiKJSVlck8gjdcLpc8uGFhYdx2221SofsD3p8r6qJEofDtt99O\nXl4eISEhfPHFFzIk01IF/nOGicFgkLnCHj16/KTA0Ol0bNmyRZ6phx9+2K/1Wt45jHvuuecnf8/l\ncklDRIRNfYmgoCCuvfZaGQ4TQvbkyZPExMTQpUsXxowZA3iYhampqURGRsoz7Y/6P1VV2bt3L6tW\nrcJqtcpw3oMPPihDod7n/Mznd+vWjczMTPbt28fHH3/M8OHDAd+wC3U6XbNapqqqKoKDg4mLi2Pz\n5s2AJ+XQtm3bVmXQGY1GybZ0uVxUVlZKA14gKCiIwMBAysvL5fmrr6+XdYp+X2OrPKWVoKoqBw4c\noLCwkKCgILp27QpAbW0tb7zxBtnZ2fKQDBo0CLPZTHBwsDy4vq7WFp8lmDiqqkoFFR4eTlZWFrt3\n7+bUqVPy948ePeo3RRAQECBZcHq9HqvVyvLly5k+fTrgOYyC3CFi+P369ftRIt5fcDqdFBUVMW/e\nPBYvXgx4hJ7IIwwcONBn6/ilPf4loa6qKlu3bqWiooJ+/foBHoKCPyGESU1NDaWlpWdVsk6nk5Mn\nT/L9998Dnnfar18/n+cOJ02axIYNG1i9erXMq7pcLqqrqzl16pQ80wcPHiQgIIDGxkYSEhIAj1fl\nK8NQeJXV1dXceeed1NTUEB0dzVNPPQXAFVdcgdPppLa2VgrVwMDAHxkdBoOB0aNHc/DgQQoKCqRx\n6wuavk6no2fPnqSlpQGe91hSUtLs83v27CkZsq0Fu90uPV1R8tGlS5dmv+N2u1m7di1Hjx4lIiIC\n8FDSk5KSWsXLu6gUlMPh4Msvv2Tt2rVs2rRJXtKTJ09SWFiIyWRiwIABALRr147KykrJXGvNNQJ8\n9913LFiwgMrKSlkbFR8fz7hx4/y2HoPBwC233AJ4aOZ2u52NGzfKcEaPHj2wWq2Ul5fLCvvExES/\nW3UinFddXc0//vEP1q9fLwVPYGAgycnJXHrppRdUayWXy8U999yDTqfjb3/7G+D/9lki7Go2m1m2\nbBkjR46UZAXwCJPGxkY2btwoE/JVVVVccsklPhcmERERvPjii4CHEQrIULXD4ZBRgR07dqDX6zEY\nDAwePBiAu+++m9TU1Bafc1VVJelp7dq1FBcXS2Omf//+gIfSLZShd/RE0KcFvBmGJSUlsguNL2jw\ner2eyMhIJk+eDMBHH31Ebm4u5eXlsq6wQ4cOrd6pxGw2N2Nbbtu2jT/84Q9ynxwOBwUFBeTk5FBX\nVyeJL5s3b2b48OGagvqtKC0tZenSpZjNZoxGI99++y2AtIZ69uwp6cs9evQgNjb2R3UG/shJidBj\nQ0OD9EwWLFhASUkJqqpKy+SBBx7wubV7Jjp06ADAzTffzL59+6ipqWHOnDmAZ58SExPp37+/zJ/5\nOjx0Noj3U1paSkxMDElJSZLOHhoaSnR0NJWVlTgcDinU/LlHv4aGvmLFCoqKiujZsycjR470+5oA\n+U769OlDRUUFK1eu5PLLL5dCVLDYcnNzZYi2c+fOfgthp6Sk8M4778hn1dbWUlxczK5duyQztbS0\nlLq6OiwWiyy76Nq1K1OnTm1xqM9sNrNu3TrAI/QFA7Rr167yToeHhxMSEkJAQMCPmHHeCspoNNK3\nb19Z1lBWVgb4RkGJzxcF1QMGDGDHjh0UFxfLsFl4eHirK6jIyMhmLZX27NnDjh07pMFaUVHB8ePH\nf9Q3MCgoqNXWeuGYpBo0aNCgQYMXLgoPSoSI/vnPf9LY2Cjb6YtwmsFgICIignHjxsmiVFFv5M1c\nA99awd6MuCVLlvDhhx+yd+9ewOM+u91uIiIiZMeLGTNm+D2c5t2wsrKykhMnTshwhk6n4/jx49TW\n1soQn2jp48+WUOKzO3XqxHPPPYder+fkyZMArFq1iqysLI4dO8aRI0ekB+zP9XiHf84kcIAnB3TX\nXXcBHhLHL70z8XktDQGKfMVLL73Ejh076N27N2FhYTJEXFVVRVNTEzU1NbJm5YYbbvAruSUiIkIS\nSlRVZeDAgVx11VXs378fgC+++IKsrCwKCwvlOdu9ezfDhg2TDZPPFVVVVbILSX19PSaTiV69ejFk\nyBBZACs6jZwtv+z933q9npSUFFwuF1arlV27dgE/eK0thV6vJy4uDvB0bGloaGDRokXynE2ZMkXm\nqFoLYWFh9O3bF4CcnBx0Oh1vv/22zNVVVlaSlpaG1WolNDRURnqmTJmiFer+Fog4qt1ub1agKBRU\nQEAAo0aN4pZbbpG0YG+aq/c8JF9uvAhdvf/++7z88svU19dLIRcbG4vD4SA4OJibbrpJrsmfUBRF\nhhg3bNjAiRMnZIIbPDRpVVWpra3liy++ADx5hUmTJskiTH9AhBlCQ0OlMBcJ9cjISBRFoaioiM2b\nN0v6tz/zPd4d1t1utySOiI4RCxYswGazMWHCBK655pqzhowEfNkFQHxOr1696Nq1K6qqUlxcLDtx\nxMbGEhsbS/fu3WV3EtEnz5/w7lyuqirh4eGSbZmamkp4eDhbt26V4aw9e/awcuVK7r333hb1U9y5\ncyerV68GPO2XgoODiYyMJCUlpVnPxl97VmJiYnA6nbhcLski9aVMEPcnICCA2tpaampqJLNxyZIl\n0uBoNeFvNPLhhx8Cnm7uMTExHDt2TPbEbN++PYqiyDZIwrj3Nxmo2Rpb7Ul+hDiAQ4cOZcuWLbRp\n04ZTp07JdjXiwohkrfg7/p6HIoRcUVERcXFxREZGyiaeEyZM4OWXX2bbtm1+bXoqIJrXzp49G4D8\n/HwcDgexsbHcdtttAEyePFlW04tOCu+88w6LFy/mnXfeISMjwy9KypvtKBppCmtbVVXZ7LM1hK2o\nB7FYLICnJVVeXh579+6VQmvLli1069aNxx9/nMTExLO+v5+z1lsKvV6P0Wikrq6OmpoaaeW3adMG\nu91OWloaPXv29PlzfwpnKmbvcoqUlBT++te/Mm7cOCkMN27cSE5ODlarVZZinMs6u3fvLlloTqdT\nlo6cOHFC5lF+y+fu3LlT9hEUNHVfygjxOQEBAZjNZrp06SKJB5s2beK///0vd955p18axP4URN/B\nV199lWXLlgHIEoEBAwZQVFTE7t27cbvdknrfGnlpgYtKQU2aNIn9+/dTVlZGWVmZPPwhISFUVFRQ\nVlYm64C8w3v+mvcjrLhbbrmFm266ia5du8rkv91ul4l/EY7wJ1RV5eDBg+zevRvweAadO3fmqaee\nkgWU4NmXuro6tm7dCnjCWadOnWLWrFm8//77MmnsCw/mbB5HU1MT+fn5fPXVVwD873//w2w207Nn\nT6666iq/XA6xDkVRqKmpkS18wGNllpaWYrFY5MC7+vp6xo8fT3p6+lk7mrcG3G43VVVVklQCnnNu\ns9mYPHmyZPz50wg7WyPRM2EwGAgNDaVDhw4yRARw/PhxysrKftRe57egXbt20mix2Ww4nU727NnD\nwoULJV06NTX1V31/l8vFs88+Kz0G0dbKlxDrCA0NZcqUKWRlZfHll18CHsLH0aNHaWxsbNVed+I5\nycnJTJ06lfr6ehmKDAoKorGxkfLycux2e6tNXfCGRpLQoEGDBg0XJC4KD0podJPJRGNjIyUlJbjd\nbulBiQJUh8PRLC8lLGfvEI0vrQNhhfft21e2YfK21hsaGtDpdDIU6E+oqsrRo0d/ZFmfPn2anTt3\nAlBcXMyBAwfYsmWLbIAqfq+qqoqKiopmHqgv1qQoiszLNTQ0cOTIEb799lu2bdsGgMViISMjg4ce\neshvxYHeYyLKysqIi4uT1n5gYCCDBg0iNzdXNvbs0KEDjzzySLOmttC6lmVJSQlms5m0tDRp2YrQ\nX2BgoOzu4C+4XC5cLheKoshwnohKeO+DOD+VlZWS0OB0OjEajZLcca4IDQ3lD3/4AwCffvopNpuN\nqqoqli5dKu/Z008/TVxc3Fnnh4nyD4BHHnmEY8eOAR5vwrug3dcIDg5myJAh6PV62UnC5XIRHR2N\nxWJp1nKo1ejc/z+Jo02bNs3OdGpqKlarFZfL1aytV2vholBQAgaDgdjYWMrKyiSrRyA6Oprg4OBm\nOShorpT8dRjOFpZyOBySXSS6ifsTOp2OgQMHytlBFouFwsJCnnjiiWbzfAICAnC73VLoxMTEMHLk\nSG6//XbS0tJ8FmITBkNdXZ0UDNnZ2Rw5coTS0lLZiaB///7MmTOHpKQkv/Rv84bRaKRz585yfeA5\nJyEhITJxDJ6eiW3btvW5QfNroSgK+fn5lJeX06tXrx+FGL1HJPjr+WVlZWzevJnMzEyZxxDv58xZ\nUKdPn+bpp5+WeU2n00nbtm2bhfzOBQaDgVdeeQXwhPiWL19ObW0tDoeDVatWAXDixAni4+NJTk7m\n+uuvBzzMPKfTySeffMK///1vwJMnVhSFNm3a8L///c8vc6oERI6uS5cusr9dU1MTJ0+exGw2S9kg\nfre1IGpCvRW4qIGKiYnRFFRLERQUhF6vp6amptkgtbq6OtkoVuSAzkfeAH4QfIISHBMTIxlr/oRe\nryc9PZ0nnngCgLfffpujR49SW1srrUjw5DYSExMZMWIE4CkezsjIIDAw0KfWpKIolJeXs2rVKmlF\nlpSUEB4eTnh4uGQ23nPPPcTExPymd/Rb8y5CGKiqisFgQFVV6dUKckmbNm345z//CfAjpXA+EBgY\n2GysxpkQBCFfNov1njK8YsUKli1bRnp6OnfccQfgGUMSEBCAwWCQ5IV169bx4osvUlBQINm2MTEx\npKSknHX+0G+FMELfeustRowYwVtvvUVOTo5U0Lt27ZJMzLlz5wIez8tut2O1WuWawEPo+OCDDxg8\neLDfyz30ej0RERGyHdumTZsoKipix44dREVFyTKB8w1FUQgLCyMzM1NGpFoTF5WC0uv1XHPNNbz3\n3nvNQkd6vZ7Q0FDCw8PP2tG4NSHWtG7dOiwWCyNHjvTpWO6fg8lkktNzx4wZw6JFi9i4caMM8ZlM\nJkaPHs306dOlZSeUvj+gqipbtmwhLy8P8IQTUlJS6NevH1OmTAHOrcLeF+9WKK2mpiays7O5+eab\nJbvpfCsnVVVJSkoiNjb2rN/V7Xa3iHzwUzizC/iJEyfYvn277OaQmpqKyWTCbDbLEGNNTY0MWQnj\ncNCgQcyYMcMnbDXx94OCgrjhhhuYOHEiubm5LFy4EIBvvvmG0tJSbDabVEaio4XwjgHGjh3LG2+8\nIev+/A2xH4KM8f7771NWVsaxY8fk5N/zBe80xPHjxzEajURFRZ2XUfAaSUKDBg0aNFyQuKg8KJ1O\nx9ChQ3nppZeYN2+e/Hn37t2ZMmVKs8aa5wNiQiV4CmVFH77WXJOwgqKiorjrrru46667flSo7O96\nLPGny+ViwIABssV/eno648aNo3Pnzq3Sc+9MiPyJ9zNdLhc9e/akW7durdpU+Oeg0+mIior60XpF\nDZfVav3Z4uGWIjAwkBtuuIHdu3ezaNEi2beurKwMl8vVrHRDrC8iIoJp06YB8OSTT/ql95zoxTdw\n4MBmzWJramo4cuSILFI/duwYNpuNvn37Mnr0aMATnvTX6J2fW68oPB80aBCHDx+mQ4cOREdHn5cI\nj3hXIspjsVhYt24dERER561Zs84fB9gHaPGizlY8eL6hqipHjhwBPC1yTp06xcyZM3nwwQeB1i2A\nOx/wfid2u53S0tJmYxASEhLkoMlf877ERTqTjdnS9amqKgt1t2/fTrt27UhLSzsvIY6zQVEUbDYb\nFRUVREVFydyAYKbW1NTIvREtvXwNYWx9/fXXfPrppwDs37+fqqoqObgQkMLtlVdekR0IzneI9EKE\naLEUGBjY6ooSmjNqRU66oqKC559/nvz8fN544w2pTM9hbef8ZS5aBXUhwlvw/e9//6OoqIj77rtP\nFupeCEq0taAoCg6HA6PR2EzBnG8P1+VyYbPZ5Oyl+vp6evbsecEopwsNiqJIurhOp5MjL0SxZ0BA\nACaTSdu/3xG8c1B2u122OmoBNAWlQYMvIDwPAcHs06BBwzlDU1AaNGjQoOGCxDkrKC0YrEGDBg0a\nLkhoCkqDBg0aNFyQ0BSUBg0aNGi4IKEpKA0aNGjQcEFCU1AaNGjQoOGChKagNGjQoEHDBQlNQWm4\nqHBmHZMGDb8F3m2aNJx/XJTl3WceMLvdTnFxMSEhIXI8tlZ8eXFCa6Oj4Vzgbdj8X+roIqAoCm63\nm4aGBnJzcwFPv9CQkBCGDRvG0KFDz4vMvKgUlNPppKKiApvNhtVqlXOGFi9eTE5ODnq9nj59+gDw\n+eefn5cBXGcOczvzYngPDNPggWhBZLfbsdlsgGc0iKqq2Gw2OSXZaDTKgYxaax0NvwQxNBM8A0RF\nS6b/S/fPe8bX6dOn+eyzz/jmm28AOH36NCaTif379wMwdOhQoHWN+4viFgshb7VaWbx4MeXl5QQE\nBFBcXAxAbW0tjY2NuN1uvv/+ewDWr1/PDTfc0CqHUSgit9stB+HV19dTVVVFQkKCHNktJpJeSBfE\nW6G2ZF3eDUR/y3PtdjuLFy9m7ty56HQ6ObI+MTERt9tNXFyc7JsXFRXF/fffT5cuXeRoeg2tD+9R\n6g6HA6fTiU6no76+HvDM+PotTYH9tcbGxkY2btwIQI8ePUhOTm5Rx/rfOijz5z5HzK7ybrXlj+iA\nkJ3FxcUsWrSI0tJSObfLZDJRU1PDjh07CA8PZ/DgwUDrKigtHqJBgwYNGi5IXBQelLBaXC4X9fX1\nmEwmrrzyShITEwFPuOfJJ59k8eLFMvSTnp7uN+tNWEAiHHXo0CE2bdrE1q1bOXnypPydmJgYunfv\nzhVXXAFARkYGKSkpmEymVrNSRFdxgIMHD1JaWkpZWRmlpaWAx7Lq2rUrd9xxx09OcP01+K3fR3hK\nK1eu5JNPPqG0tJS0tDSGDRsGwHXXXUdMTAyNjY188skngGe0hNPpbGnn5Z/EmblN4RmL8eY2m42Y\nmBifegbC8xZTaeHC87TFvrhcLk6ePMk777zDmjVrAM97VFWVwMBA6RUkJyfzl7/8hSlTpvjtXf3c\nOp1OJ5s3b2bOnDlyNPy9997LxIkTW7Snvngfbreb2tpatmzZAnim/44fP57IyEi5V76UDUJGbdq0\niYMHD9KtWzcmTpwIQEhICA8++CD5+fnk5uZKOdGac9EuCgUl3NTKykqKiopISEggMTFREiJ0Oh2T\nJk1iyZIlcpNDQkJ85pJ7w+12U1RUxIoVKzhw4AAAW7dupby8HKfTKWf3REREAJ6hYCtWrADgq6++\n4oorrmD8+PHy9/ypRF0uF3l5ebz99tuARxm5XC7q6uqoqqqSv7t//34GDx7M0KFDWyW3oyiKHEP/\n9ttvk5qayrx582jbtu2PwhyqqtKrV69mP2vJBVYUBVVVZSi2urqa6upqzGYzx48fByAuLo60tDRs\nNhtz585l9+7dAAQHB/Pvf/+bzp07t+i9qaoqR1iUlJSwYMECTpw4weHDhwHPGevUqRODBw9m3Lhx\nJCcnA54zbTAYWlV5iTwgwKJFi3juueeoqqqS7yk6OpqwsLBmBtvBgwf5+OOPGTJkCGlpaYD/iQmK\nokij65NPPmH16tVUVFTQt29fAIYNG9Ziwet0Os/ZOFFVlby8PF566SXcbrccy6OqKg0NDcTExDBl\nyhTA8559FUoUxlVhYSEhISGMGTOG7t27A56UiVjL+QrHXhQKSlhmq1atYtu2bQwdOrSZtasoCnv2\n7EFRFAIDAwGkomoJvAfciT8XL17M448/zqlTp340VdRgMEjhqaoq4eHhWCwWampqAM9E0pKSEvr1\n6yeHu/kLTqeTzz77jDlz5sg1tW/fno4dOxITEyPXVFNTg9FoJCYmphkF15+Hta6ujr/85S+AJw4+\nd+5cGRc/E2JfWwJvD6CiooI9e/bwn//8B4CcnBwcDgd6vV4+JygoiJSUFIKCgti7d6+0wqOionxi\n9KiqSl1dHQCvvfYaq1atora2Vp5znU7HgQMHWL58OU888YRcl7dHPnbsWLmm+Ph4vw3BU1WVgoIC\nAF5//XXKysqIjY2V+YoxY8bQv39/8vPz5WDD7OxsTp06RUlJCR06dJDfyR9QVZWqqipeeOEFNmzY\nAEBTUxNxcXEMGDCAu+++G2j5YEdVVdHr9b/5/QvjevLkyWzatImQkBAmTZrEwIEDAc+ZTEhIwGw2\ny7yeLyEiFXv37pWGvVDUwuB3u900NTVpLL5zhdjkzz//nNraWsLCwggNDZUXGjxCJTQ0lLCwMIBz\nOkxnwuVyYbFYpLLbv38///jHPygqKpIHFjwWvdFoJDIyUjJhhgwZQlBQEIGBgSxbtgzwKE2DwYDF\nYpEH19eHQgjjgoICXn75ZUJDQ6VL/4c//IGIiAhMJpP0oIqLizGbzbRr105a5/6Eqqq89NJLFBUV\nAfyscvIVxDk5dOgQ//nPf8jKypKhWKPRSFJSEr169SI9PR3wKPK4uDj2799PTk6OvNCpqam0a9eu\nxXuk0+mkB2Wz2aTnLT5Xr9dLwo3T6ZRnpaqqis2bN7N161bef/99wEMmefHFFxk4cKBcp6/foTgr\ner2e3r17M3PmTEaOHAl47p0YWijOntFoJCMjw69hdrEn69at4+9//zvV1dV06tQJgIkTJ5Kamkpt\nbS2ZmZlyTS2F0+lEr9fLEOwvweVySUO0rKyMkJAQxo0bx1NPPSUjKI2NjTQ2NlJSUiJDfL7aM6G8\nxVp69epFdHS0fE+rVq2ioaEBVVXp06fPeWHGXhQKSlj75eXlGI1G4uLiqK2tJSQkBPCEXm688Ua+\n+uorKfh27dpFjx49WmQ1ud1uSktLJRPoiy++wGazERwcjF6vlzR2RVEwGAx07NiRnj17AtClSxdC\nQkLYs2cPZWVlgMelFiE2ccF8TTsXh+/ll1+moaGBadOmMX36dOCHkeFut5vGxkYATp06RefOnVtF\nOYFnD+bOnSunDN94441+f6a3Z2IwGAgJCWHGjBkATJkyhc6dOzfzQIQnaTQaZSgQYPr06VKwtAQ6\nnU4KrpdffplRo0YRERFB165dAU/YzGQyYbVacTqdVFdXAx5a8Jo1a9iyZYsMZ1ksFmrC1BVyAAAg\nAElEQVRra/327nQ6HT169ADgrrvuokuXLgwePFhGKnQ6HU6nk+XLl7Njxw7Acx9Gjx5NfHy8X9bl\ndrt54403APjnP/+JXq/nkksu4d577wU8SvvIkSOUlZVJodvSdTQ0NFBQUECnTp2aedtn+1xVVTGb\nzXTp0kUqiNjYWL788ksGDhwojWfxu9nZ2VitVr8UoQsZOXDgQBITE7Hb7dLgX7BgAYqioNfrmTRp\n0nmpMdRYfBo0aNCg4YLE796DUlWVdevWAZ56J6PRyP79+xk3bpx0ifV6PYmJiSQkJFBYWAhAVlYW\nkydPJjo6+pytJ71eT3p6uqy3uu666zAYDAwZMoQOHTpI68xut1NWVkZ1dbW0QkJCQqipqaGyslLm\nMCwWCwUFBezatYvOnTsDSC/MV5amCEceOXKEmJgYxo4dK61+p9OJ3W4nLy+PVatWAZ4QTf/+/VvF\nvVdVlRdffBFA5gtaw2oToa+AgADi4uK45ppruOuuuwAPIeLMvRckis8++4za2loZOrrxxht9tl6x\n34mJidxyyy3N6mHODPsKooEIxUyZMkWuOTMzU5Jb/OGt6HQ6WXM2ZcoUmXQX1r6iKHzzzTfMnj1b\nnvM+ffpwww03+JwNpqoqFouFRx55hM8++wzweJsTJkzg+uuvl17dli1bWLVqFZ06dWqWE27J/ohQ\n5unTp0lLS5Neuffnl5eXAx6veN68eTgcDvr37w94WHQmk6mZlw6e8Onx48cJDAz0+R3U6XTSg7Lb\n7Rw/fpy4uDhJ7jpy5AgAYWFhcp2tjd+9glIURXaMsNls6PV6CgoKCA4ObuZm6/V60tLSyM7OBiA/\nP5/s7GyuvPLKc74ogYGBKIrCZZddBsDo0aN/MhQmKNCCnVNaWkpVVRVVVVWYzWbAE95qampiz549\nMkkaEhIiD5EvINz3mpoaYmJicLlcMpx34sQJ9u/fz9q1a2Uo9JJLLmm1okq73c63337LiBEjJDPt\npyAEoC8Za1FRUfTs2ZOkpCRiYmLk558NRUVFrFq1CoPBIFmQ/siV6fX6X50vFcZacXGxLP6+++67\niYqK8uu7E0o5MjKSuro6SkpK5Dn77rvveOaZZ2hoaJD53zfeeKNFhuHZoKoqdrudt956i6+++oqE\nhAT5rMzMTOx2O08//TQAa9euJTAwkISEBElGCQ0NbdF6TCYTGRkZKIoimaDgudMVFRX861//Yu3a\ntYAnHJiYmMjrr7/O1VdfDTTPgSmKIpX5u+++S3l5eYsp8D8FIXsqKirIzs7myJEjVFRUAMgC65iY\nGMk6bm387hWU0+mU9FtxMJqammQ+RUBYveLiHD16lAULFjBo0CDZHudc6aHCA/k561mn0xEQECB/\n59ChQyxcuJADBw40o74bjUaamppYuXIlAB06dJDWky8O6KlTpwAPkyk8PJyPPvpIepoWi4WAgADs\ndrtM0rdr147o6OhW8WROnz6N3W7nnnvukT8T71BcfLFXer2+mcXZEojPCAsLo2PHjlK4ez/f+zlu\nt5vXXnsNi8VCt27dJPHFn3v0a76n1Wrlvffeo6GhgcsvvxyAcePG+Z195U3eqK6u5qOPPpLW9759\n+7BYLJhMJubMmQNA//79/bJXVquV3bt30759eyZPngx4OkTodDpmzZolIy2CDn7o0CHeffddAJ54\n4gmpQM8VIt/mcDjkPfvPf/7D119/jdVqlZ8/YsQIXnzxRZKSkuQ+eOeczGYzjz/+OABr1qyhZ8+e\nXHnllT73oLzZohaLBbPZzMGDB6XSEjnZAQMGnLcel797BWUwGEhNTQU8l0EUTxYWFjajajudTlwu\nl6RqulwurFYrZrNZKqhzff5vgSB0LF++nK1bt0pSB3iKh4UgFsSJvLw8jEajT/oGKopCTk6O/Pfq\n6mq2bdsm6x569+5NUlISZWVlkrF22223+azu4qcgwiFHjhwhOjqaqqoquU/h4eEoikJtbS1FRUXS\nsxs5cqS0klsKbwWVnp7ejM7rdrt/dDmrqqr44osvUBSF2bNnt2qx6dkghNvrr79Obm4ugYGBUsD5\nmwHpDZfLxffff8+6detkOMvtdhMWFsbDDz/MTTfdBPivVY7RaKRPnz6YzWZpZBQVFTF//nypnMCj\nIMePH8/Ro0dlRGX16tVMnTq1xYK4sLAQm83Gv/71L8ATlXA4HAwdOpT4+HgAunXrRm5uLk6nUzYT\nMJlMKIqCzWZj/vz5fPXVV4BnTzMyMggLC/NLzea+ffsAj3Jv06YN8fHxMspjs9mw2+307t0bt9vd\nKiUmZ0IjSWjQoEGDhgsSv3sPymg08uabbwIeC7yyspLw8HDq6uo4ffq0/L1Fixaxd+9e6SYHBwcT\nEhLSouI30X5GWBS/ZFk4nU5eeOEFwNOs1u12k5qaSkZGBgCXX345paWllJaWSiumurpaWi++sFxE\nQnvo0KGYTCa6du3K6NGjAU8LmrVr11JXV8cdd9wBQFJSkl8tJu+uCSdOnKBz587k5eVJS/aSSy7h\n2LFjfP3119hsNln576+Ef0hICIqiNHun4t9F3mvNmjU0NDQQFxfH0KFDz2vLIVVVJfHn1VdfRVEU\nJk6cKOt7WmNtYl9KS0tZt24dlZWV8mehoaHcfvvt/O1vf/NrixydTkdwcDCXX345y5cvl/TtwsJC\ntm3bhsPhYPjw4YCnO0l0dDSLFi2SXUCysrKYNGmSJFKcC9avX09VVRXffvst69evBzzff8CAAYwe\nPZpBgwYByEkLGzdulPVi/fr1w263U1RUxKFDh2R0oG3btsyaNcvnITZVVamtrZUEr4iICNq3b4/J\nZJKF1zabjdDQUJKSknC5XLKkwl9F32fD715B6XQ6UlJSAPjss8/47LPPCAkJISwsTF7cFStWsHDh\nQmw2m7wkwcHBJCQktEgAC5f31xweRVHIysriyy+/BDzKKi0tjXHjxnHllVcCHmWwceNGdu3aJdeZ\nm5vLyJEjfXJAdTodQ4YMATykjejoaOLi4qTSPnXqFF988QUOh0Mmb1ujMFdg/PjxjBkzhqCgIKKi\nouTPGxoauPHGG2lqapKho7q6OmJjY30eLvq5zhQi//Xaa6+hqirvvfdeiwTauULUYbndbrKzs/nz\nn/8MeAymSZMm8f7777davzTvcPS7777LwYMH0el0MuzZt29fnnjiiZ/cJ1+GjQwGA3369KG0tFTW\nXB0/fhyHw8GAAQN48sknAQ8z1u12c+zYMdlh3bvX4bkiJiaGY8eOkZubK3PdmZmZ3HPPPXLMD3iK\nby+//HJycnJkekKv1xMWFsaWLVvIycmRzMznn3+e0NDQFq3rbDCbzcybN0++u549exIeHk5ubq7c\nE9FnMicnh0GDBkk5+2sLkX2B372Cgh9i2t27d+e+++6jrKyMyspKdu3aBcDGjRupq6tDr9dLNkpy\ncjLJycktyh/82uJVt9tNZWUlS5YskfkuvV7P1VdfLYtAwWPt7dq1C6PRKAV0ZmamT1l84rO6dOki\nyRfC2t26dSt79+7l1ltv9cul8IYQsqqqSmEqlKX3BdDpdPTr1w/wKHWRfD569Cjt27dvtfYrqqrK\nQW7FxcVEREQwatSo8+I9CTr1Bx98wOzZs+W4ke7du/Pmm28SHBzcap5TTk4OH374IQDbt2+XJR3i\nTI0aNYrIyMifXI8v16nT6QgLC6Nnz56yTCIwMJCMjAymTJki87ilpaV8+umnLFu2jI4dOwLw+OOP\nt5iEINiC+/fvlwqmR48eJCUl4Xa7pdIyGAzExcUxbtw4mSvT6XQcO3aMnTt3YrFYuP766wFP1xJf\nv0ubzcajjz7K119/LT21oUOHkpSUxOnTp6URKKJLouj7fBAlLgoFJSAIB5GRkSQnJ8sXm5CQIFl8\n3rVRonnlueJs9TGCaVZbWyutk+3bt1NeXk5wcDBXXXUV8ENjz2+++YalS5cCSMp5enq67No9atQo\nn7F3dDqd/CyDwSAPnNiHN954A51Ox0MPPeTX3mjiTxFKEz8LDAz8keXvHWILDAyUl/zEiROS3t8a\ncLvdfPzxx4BHMF977bWtSkA4E0eOHGHOnDmUlZVJJX3TTTcRFhb2kyFKX0JRFE6dOsXSpUslLbl7\n9+707t2b3NxcefYzMzNbtUWOTqcjNTVVdmwxm83o9XoiIyNZuHAh4Kmxq6qqolu3brz++uuAJ5TW\n0n0KCQmhe/fuXHfdddIQraurk+UjwugLDg6WtHZxBwsKCsjKysJsNtO7d2/ZS9GXBpi4Z+vXr2fh\nwoU4nU65zvj4eBITE+nXr5+sQRS1dwEBAURGRjbrDtJa0EgSGjRo0KDhgsRF5UGJEc4ulwuDwSDH\nbTz22GMEBwezfv16mQPKz8+nV69eLR6L4P2ny+WivLycBQsWsGbNGo4dOwZ4rM3k5GT69u0r3eY9\ne/aQl5dHQ0ODDIeIxqRpaWmyjuXnwiPngrN9lqBul5WV0bdvX7lv/oTD4UBRFFwul/SahDX5c99X\ndCoQHmlrwWazSepvXFwczz777HmrDdHpdDQ2NmK32zEYDDLRLmjc3pEB0UHf21r3xborKyuZP38+\nVVVVkpAxduxYKioqqKyslOd82LBhrWpxi/yXKDGprKyksbGRxYsXS9KU0+lk8uTJ3H777TLE5Ys9\n0ev1REVFcdVVV8l8ZWlpKYqiEBIS0ix6IyIHtbW1ck0mk4k2bdowdepUWTPlj73bsWMHdrsdRVGk\n92swGIiPj6ewsFCG9IQcbdeuHZGRkVqIzxdQFIXi4mJKSkpkh4S+ffsSExODyWSSxAmbzcbJkydb\n3IDRewSFqGYXHSKEoFBVlZKSEsrKyqSAFZ2o4Qc3vkOHDtx0003MmDFD5qD8dSjEwXe73SxYsECu\n49577/VrSEbsVWNjI01N/197Zx4dZXU+/s9smck22ROyA2EJS9j36JeACMguiAsVlyO0FfV4LKVY\nD9pzqq0Vi0stVU+htSpqQdyKFlxZBIOQoCECISwRspCVEJJMZv/98f7uZYJRgcyEwd7PP/akIXPn\nfe+9z/48LezatUsWuooGsd+H0+mU8Zabbrqpyw6M1+vlyy+/lG6rUaNGya7P35dd6Zv8EaiMQ4vF\nQmxsLI8++iigBelFFwKxr0XWpsFgkO5R3wLnS1mb2+3m6NGjHD58mMjISFkUGxMTQ35+PrW1tcyY\nMQNA1vh1NUJAbd68mWPHjuHxeGQsc8mSJfTo0QOj0ej3PaTT6dq5qnv37v2dPeL1ejGbze1G/kRG\nRmI2m8nJyWHMmDEB3du9e/fGYrHQ2toq98Rbb73FkSNH2LlzZ7tC3cTERG666SZCQkIuSIH0Nz8p\nASViLMeOHeO9996T2snRo0cxm81s3rxZCgiXy+WXynFfTCaTnBba2Ngou5w3NTXhdDrleuBc76+o\nqCjy8vIAeOSRR+jbt6/fOiT8GOKiEa2isrKy5HTfQOFrbe7du5f8/HzZpTs5Ofl7fe5ut5sTJ05I\n7W78+PFddlC8Xi9/+9vf5AHNyckhJCREZtL5rlnE1sT39P3//Lle0XVk+vTp0oIRrbdCQ0Pl5ecb\n6xM/E1bVpV6CLpeL9evXU1ZWRq9evTh27BigKRBHjhxh5syZ3H///UDXB9bFAMUdO3YA2uwpj8fD\n3LlzWbZsGaBl8QV6Xb7vuqP37vF4ZNcJse6MjAxSU1P9mhTV0ZpuuukmampqWLVqVbuWVNu3b6e+\nvl4qzgkJCTz66KP07t1bDSz0B6IWIiEhgdLSUqkJ1NXV0dTUxLfffitdD6GhoYwePbrTG/X8IHRo\naCg9evTgmWeeobS0FNCaU7777rt89dVX8vN1Oh2pqaksWLCA22+/HdCSOboqK83lcnHq1Cn+8Y9/\nyEMye/bsC3abXWpdlnje8fHxDB48GLPZLF0fLS0tMnlE/J6Y6FlYWEhBQQHz588HurZDQnNzM3v3\n7pXrHDlypExL9k1I8G1b49v0MxAH22az0a9fP+644452iSV6vZ6wsDAZ0BbWsMfjaaeQdSZ5wuPx\nUFVVxaFDhzh+/Lj0SqSnpzN58mRmzJjhl7Ejl4po2QVaY9qRI0eyaNEi6ZW4nHVr4vPFvhCZhTab\njZCQEGw2W8CFp9ls5u677yYpKYk1a9YA56Zpx8XF0b9/f0Ar+5g+fXqXKcwdofN1RQQRnVqUx+Oh\nurqar7/+GoDDhw9TWFjIF198IVv4TJ8+ndtuuy2gTRDFs3U4HLS1tVFTUyNrDMxmc7tpp9A1B0es\n6fTp0zz88MPs3LlTptr+/e9/93sTzx9ah81mo6ysTH7e4cOHZX87ccF89tln5Ofn43K5WLBggazP\n6kr33pYtW7j33nvp06cPoPVXS0hIkNaJuIzF93A6ne0aFYsLyV9rdrlcFBcXs3//fmbOnCn3sGgq\n6+t2Fj/zLUgXLqZLpbW1lRtuuIE9e/Zgt9tlLc+qVavIy8u7rMJJuNnFwEmbzSZdWpdbMPni8Xhw\nu92yzVdjYyPFxcVkZmbSq1evLjuD4vPFvCmXyyWVv5CQEH/NgbvkP6Cy+BQKhUIRlPykXHwCvV5P\ncnKyzEYTtUddjdA8hBsrKiqK8y3Wy6XV1dTUyE7TYtaL1WrtsvWIIH+fPn2kFhcaGkp1dbWMF4Km\n5c2ePZvhw4eTkZHR5TENMeF4xIgRMtYjGne6XC5qa2tlJpjvdFZfd58/rSfQLLTIyEgmT57cbpS6\n+K/vlF+hARsMhnZxsfNjZxeDXq9n4MCBsrnvwoULAS0ueLkb54o4tGjM6s+u9/5EWNbCsrVarQwd\nOpQTJ04EzC3c0RrEnvXt4B9M/CRdfIrvR7zvffv2sXr1aurq6njxxRcBLQYWbAf5cmOz2SgqKuL9\n99+X7sUBAwZgsVhobm6mqalJdl7v3r07ERER/nKLfIfzBZDofO+bvux2u7HZbLKXY0RExHfGwXR2\nbSI9+eDBg/Ts2VO2wOmq+OmP4eu6ggvv+NKVnH/vijW7XK6gc0f6gUv+MkpA/Q8jAvw/scPgVzwe\nD83NzZw+fVpq5eLCF3E0cYbcbneXWKHiMhPxJV/rTHRMEcJCDDxUXBl4PJ6f4vtSAkqhUCgUQYlK\nklAoFArFTwsloBQKhUIRlCgBpVAoFIqgRAkohUKhUAQlSkApFAqFIihRAkqhUCgUQYkSUAqFQqEI\nSpSAUiiCjCCtTVQoupyfZC++YCHQQ+s6y/kD94JxjcFKQ0MDDz74ICNHjmTx4sV+/dsX8x5ER3XQ\n+gaKvnvB0nYo0Igzdv4z8z17oiWUeE5ihIvD4eCrr74C4O2338ZoNJKTk8Ntt91GeHh4h39X0bX8\n5ASU2Jgej0dOtD148CDr16+npKRE/l5qaipLly6lZ8+eAduEvk08L/XCEALE3w0kvV4vZ86c4fjx\n43LqanJyctD1ARNtfURLH9FjTVzE5681EM+qozWtWrWKN954g7i4uO+9JAOJEEwul0sOnauvr8fh\ncBAVFUVMTAygDdEMZOscMSAQoLi4mOrqalJSUuRYG4PBQEhISLuBd4FQiETbLpvNJodbAhQUFLB9\n+3YOHjwoB4a2tbXJsTfi2bndbpxOJzqdjueff14OPBQzpAKF7x3R0d711zPyer1y7pvL5aK5uZmG\nhgb5nKqqqtizZw8HDhxg3759gNaHMiIignHjxvHUU0+1e6ddxRXf6sh39o3T6aSyspKPPvqILVu2\nsHXrVkCbfXT+99Tr9WRmZvLxxx/To0cPwL8HRsylAaitrSUhIeGCuiqLC1loe2IoHmjzWTq7RvEc\n2traePfdd1m7dq08hDfeeCOzZs1qd/GLA+PbobsrEOs8e/YsbW1tmM1mQkJCZPdnMavm+w5LIIWF\n2+3m2muvZdu2bYwdO5bPP/88YJ/VEWIaa319PUVFRWzbtg2AY8eO0dTURHR0NLGxsQDMmTOHcePG\nERYW5vdn4vV6qays5Ne//jUAH3/8MaB15haj3gcPHsy0adPo27evfFfR0dFYLBaMRmOH3c8vdJ91\ndHedPn2a2tpaNmzYAMC6deuoqKho171dTDwODQ2Vz6S5uRmXyyXne+3cuRPQBh4GCrfbTWNjIwCF\nhYVUVFQQHx9PXV2dnDLdv39/2T3fl4s5i+dPeXa5XLz99tu8+eabFBYWAto8KrvdjtPpbKcEivMf\nExPDU089BcDChQsvdi+pVkcKhUKh+Glxxbv4XC4X5eXlACxfvpwPP/yQ1tbWdlYIfFej1ul01NXV\nsWXLFhYtWgTQbnR2Z3E4HHz77beANoF1woQJTJo0qcNpo74WYFlZGdu3b5fuiNGjR5OcnExsbCxW\nq7XTaxTPwe12k5+fz7Fjx6S2e/LkSU6cOIHNZuPIkSMANDU1YbFYmDlzZkC08O9DjJQQbgnQXA4C\nt9tNRETEd8ar+06ShcBYUnq9ntTUVAA5jTjQ+MaaysvLWbt2Ldu2bZMWg/gdp9OJ0+mU3//1118n\nNzeXJ598Umrl/nombW1tvPTSS2zfvh3Qzs8tt9zCrbfeKl2M4vdOnz5NU1MTABUVFaSkpJCZmSm9\nF0ajEb1ef9Ed9s//PavVSmhoKD//+c8BzQJ5+eWX23V9j4uLY8iQIQwaNEjuF2FRnDp1Cp1OJ70f\ngcDj8dDU1MSqVatYt24doFkwBoOB2NhYwsPDpTtt3LhxzJkzR3bSB817YDKZLvgZddTxPiIigm7d\nupGdnQ1oLj4xukVYtT169GD37t2UlZXR2NjI448/DsC8efNkjC7QXNECyuPxUFZWxq9+9SsAPv30\nU+kP1+l0cpR6TEwM3bt3JzIyUh6SqqoquRF9BZk/EGMYHnzwQQB2797Nvn37SEtLY8CAAcA5P67L\n5ZLjqV988UU++ugjunXrJseLJyYmSj++v4eKHTp0iNbWVnkYGxsbOXjwIKWlpdJdU1JSQlpaGnl5\neYSFhfn1878Pp9MpL73CwkLGjh1L9+7dcTgccsBabGysFEi+78/tdl/U4b0UvF4vu3btwuv1cv/9\n9wfsc3w/z+l0UlBQAMB9991HZWWlvMzF9zeZTPTv35+QkBA5D2rfvn188sknzJs3T77Tbt26+cVV\nXFBQwIYNG+Rl9frrrzNkyJDvXIag7XNx9qKjozl58mS7tYv1iPd7IXT0HYxGI0ajUY4tnz17NpMn\nT6a5uVkqh+K/IrYJ2jnbv38/DQ0NhIeHSwXEX3i9XpqbmwFYsWIFr7zyCs3NzfLzQ0NDiYqKIisr\ni8TERGpqagBNGdmyZQu9evUiLi4O0N7fxbzD892BBoOBvLw8cnNzqaurA7SkH5vNRlxcHBEREYA2\nS2zz5s0sX74cnU7H1VdfLX+3q5TVK1pAuVwuqqqqOHDggPyZxWIhISGBcePGMWrUKABmzJhBXFwc\nNpuN6upqAL744gvq6uoYM2aM34N+brebV199VcbAHA4HR44cYd26dYwfPx6A8PBw9u3bx6FDh6iv\nrwc0zdJgMBAZGcm4ceMAbYhgcnIy0dHRft0QdrudxsZGOXEUoGfPnqSmphIZGSl9+A0NDfTv35/I\nyMgu2ZBer5fi4mL++Mc/AtrFcfXVV9PW1kZ+fj6ZmZmAdhmfH5MTE0IDvU6Px0NlZSUGg0FqoIHE\nbrezd+9e7rjjDkCbhhwTE0P//v1JSkqS+7dfv35MnTqV5ORkqXQ89thjvPTSS9TU1EihP3/+/E4/\nI6fTybPPPktFRQW33347oMWbzj9L4nNMJpOMdfbp0weLxUJqaqpUIgMV3zQYDISHhxMWFtZhtqOw\n1O12O16vF7PZTEZGhozh+QOPx8Onn37KDTfcAGhxVdCsvWuvvRaAm2++mREjRhAeHk5rayuffvop\nAEeOHKGqqooTJ05wzTXXAJCQkNCpuVF6vV7GtcQ7sdlstLW14fF45Du02+3079+f66+/nvj4eJYs\nWQIEPnHElytaQOn1elwuF926dQMgKiqKKVOmcN111xEfHy9/LjR/X/fY7NmzMZlMWK1WvwooETj+\n05/+JDd/WFgYI0eOJDw8nLKyMgAOHDhAWVkZZrNZXrpZWVlERUUxfPhwRo8eDZy7dB0Oh18vXvF3\n4+LimDdvHgATJ04kKiqKuro6qdmZTCaGDRvWoWvS33i9XkpLS5k/fz4tLS2AJjSLi4vJz8+nurpa\nWqC/+MUvZOqweK+BdOv5Ultbi91ubzetNlC43W7Ky8tZunSptIrS0tK46qqrWLhwIf369ZMab0cp\n5k888QSbNm2iqqqKZ555BtAUts66aJxOJ8ePH8dqtXLbbbcBF57d5XA4MJvN7TIMA/nOxN8W3pUz\nZ85gtVoxGo3yZzt37qSlpYXk5GRmzpzpN3e/0+nkt7/9LatXr5ZKg06nIzU1lQ0bNjB48GAAqViJ\n5I3hw4cD2l4rLS0lPT1dWoVms7nTd5Z4JuL5m81mTp48SUFBgczsGzduHDabjfvuu4/u3btLZUJ8\nL71eL/9OoDL7VJKEQqFQKIKSK9qCEtrGlClTAE0zu+666+jWrRsGg0Ga9ELaC61XIFJdff9eZzU5\nl8vFc889x+nTp2XMaNGiRdx7771ERETINdlsNkwmEwaDQaZOV1dXEx4eTmRkpNRwnU6n/J7+xGQy\nMWXKFEwmEzfffDNwLjZRX18vtb2IiAhmzJjRJSnmlZWVTJgwgbNnz9KrVy8AMjMz+fLLL/nkk0/I\nyMhg2rRpcq0isN5VlpNg9erVAEyYMCFgz0VYsGfOnGHFihXS2gYYP348Dz74ICkpKe32b0ffPyws\njIyMDCorK2XSjthvnUHsj7CwsB91h4nSCWEVV1VVERYW1mEdW6BwuVzy+zc1NdGrVy9aWlr44IMP\nAHjzzTeprq6mT58+TJkypVMWgagxBC1euHHjRhwOh7TKcnNzeeGFF+jZs6fcP6KmzeVy0dLSIp/L\n2bNn8Xq9WK1WkpOTAQKSoGAwGGhoaODVV1+VcSmn08mECRNIT0+XCSxwzuXs9R37arYAABNDSURB\nVHqlpReoEMAVLaBAM02F0LFarezfv58dO3bQ1tYmBYQoQE1KSiItLQ3QHqjL5WqX+WUwGDoVwxDu\nvddffx2TycR9990HwNKlS78TL7FarfKzRXaa2WzG4XCg1+vb1XjY7XbpEvEXFouFu+66C7fbLbP4\nxCYsKSmRmWGJiYnymQUKETy/9tpraWlpYc6cOfz+978HNKG1dOlSDAYDy5Ytk3HFkJCQdi6GrkC8\nk5deegmAW265JWCfJWpR3nvvPTZv3gzA0KFDAfjDH/5AdHT0BQlHnU4nL8vz/3ZnMBqNJCYm0tzc\nLDNORUKPcFWB9sxcLhcNDQ189tlngOa2mjVrVpfV1Xk8Hr755hueeOIJQHOR/uxnP2Pr1q1s2bIF\ngBMnTmA2mxkzZgxZWVmdWltdXR0PPPAAABs3bsTlcpGens6aNWsAGDRoEEajkaamJhkGOHDgAC6X\ni7a2NqKioti/fz+gZdYmJiZyzTXXkJ6eDhCQOKtOp2PAgAEYjUapwOzZs4eMjAyysrLQ6XQyPOFw\nOPjggw/Q6XQy29BisbRzAfqLK1pAeb1evvzyS9566y0ATp06hdPppLW1FZvNJg9JRESEjBcIn++w\nYcO4+uqr22US9ejRg7i4uHZa+cXgdrv58MMPaWlpIT09nVtvvRU4d5mejzjIIrawbt06+vXrR25u\nrhRGra2tmM1mv29K8beioqLaaeF2u52ioiK5SQPd8sVms/HYY48BWkLGtGnTeO655+T3r66uxuVy\nMXz4cMaPHx/woPqPrRW0C9ZsNktrLhCI2Mg777yD3W4nLCxMZgxeqHCCc8XrgMxM84c1brFY6N27\nNwcOHKCiogLQrF3xfsTZs9lslJeXs3HjRtmdYfLkyTLTsisSWrZu3cqiRYvkMx0yZAgvvvgihYWF\nsozBaDQyePBg5s6d2y5F/lI+b+XKlWzatAnQzlNoaCgrVqxg4MCBgHZPbdu2jeLiYhkfr62txeFw\nkJOTQ01NjUzmam1tZfz48e3SzAOFSBARwvHUqVMcOHCATZs2MWHCBFl6MmTIEKKiohg8eLDsQuNP\n5dmXK1pAOZ1OSkpK+OabbwCtGtxXOxSb32azUV9fT0hIiEz1dDqdspJdaAazZs0iPDwci8VySReg\n0+lk/fr16PV6xo4d28710ZH7UGiXomr9P//5D2FhYYwaNUpaFcIV4m93iN1u/067IN+OESKltXv3\n7gELgDocDjZs2MCuXbsAzdJdunQper2ehoYGAF5++WUqKysZM2ZMh1l7XcnevXsBTRG54YYb/J72\nL/B6vdIqEZ+ZnZ1NXl4ecHHC+cyZM/JdX3XVVQB+0XSNRiMpKSmUl5fLMyVaGvn2vXM4HFRWVnL4\n8GHZNcFutxMTExPwbh+gWQELFy6ktrZW7uPCwkKOHj2Kw+GQgjQhIYHQ0FBSUlI6tS6n08nevXul\nOxO0i7+kpISqqipAc3GKEIAQhlFRUeTm5mI2m3nsscc4evQooFmlDQ0NhIaGSkUyUM+tubmZ7du3\ny6xil8vF2bNnqaioYO/evVKx2bt3L1lZWcTHx2O1WgO6JpUkoVAoFIqg5Iq1oIRrTNQ5wDm3iLCM\nhJvK7Xaj0+lITEyUCRXTpk2TflOhMRw8eJCMjAxSUlIuaU12u53S0lL0ej3R0dHSfeDrUvH1zXu9\nXk6cOMFDDz0EQEtLCzU1NbIThvi3JpPJby4+38amMTExGAyGdlanzWZrp1m2tbUFrFltY2MjGzdu\nlBZw37592bNnDydPnuSdd94BtHo1m81GRkZGO5dQV1tPHo+He+65B9AsmDvvvDOgTYZLS0sB7X2E\nhYWxYMGCDvvW/RgbNmzA4XAQGxsr61j84R7V6XRMmTKFHTt2yHUJ17her5dn0e1209bWRnFxsdxn\nQ4cODWhT4rq6Opn6np+fz9mzZ2VRMyDPl9vtljU9RqOR6667DqvV2ql1mUwmJkyYwJ49e4BzKfXl\n5eVMmjQJ0OrAkpOTSUtLky6+kJAQQkJCyM/Pp7a2VsYNU1NTGTNmzEUVMF8KXq+Xp59+mpKSEumu\nS0hIYObMmSxevBir1SpDERUVFRw9epTevXt/p9mtv7liBRRosaWJEydKQXDmzBm6d+8uC8tEh4b8\n/HwKCgqIioqSGzc7OxuPx0NjY6N0J5lMJiwWS7titYvBZrMRExOD2WwmPDxcZjq1tLTQ1NREWVmZ\nrBDv1asXVquVP//5z7JVU2RkJAkJCeh0OuliE+5Gf3Y1hnMJETabTbpeQkNDOXLkCB9++KF0MbW0\ntHDmzBni4+P9vgFFZpO44CorK9m4cSPh4eHSHaLX6zGZTFx11VVdnhThi81m49ChQ4B2mQwbNiyg\nnyc6StvtdiIjIy+6IFi80xUrVuD1epk9ezbdu3cH/CPcdTodmZmZpKSkyHiJ3W6XCT6+HSQKCgo4\ne/asrEu86qqrAnLhioLYG2+8UbodRUFrSEiIrFkTrkiTySQTD0aPHk1ubm6nXel6vZ677rqLr7/+\nGtCSL3JycliyZAn9+/eXny/2su8F73Q62b17N7W1tVJIREVF0bdvX780iv4h2traePPNNzEajbJB\n7urVq8nOzsZkMuHxeOR9FhsbS3Z2NlarNeAZtFesgNLpdJjNZiZOnMiECRPkz4xGoxQuwhIaMGAA\n2dnZHD9+XGosIr3bbDbLHmUnTpygubmZyMjISwr6mUwmsrKysFgs9OjRQ1aMNzc3U1tby86dO6X/\nPy0tjaamJj755BP5WVlZWWRkZBATEyMPk0il9gfCagHYtGkTRUVF2Gw2GUtJSkri66+/pri4WK4z\nOzubhISEgAzRi4uLY/HixfL72Ww2RowYgU6nk0rHF198Qd++fcnMzAy4FvlD/Pvf/5YWQHJysvS9\nBwKdTictEKfTid1u5/Dhw7K7yI9ZH62treTm5gJahmRkZCSPPPKI359fWFgYw4YNk0kSO3bskJ0t\n4uPjAS3Q/t///hen08nUqVMBAvLsnE4nq1ev5qGHHsJut8s7QCRI+XbzjoyMJDY2lvT0dCZOnAho\nQjM+Ph6DwdBpayA5OZkXXngB0KyNnj17ylgyfP9lfvr0abZs2SJLUADy8vIIDw8PWFKQ8DLdc889\n1NTUMGbMGF577TVAuw8Eer1erl8IeH9kg/4YV6yAAtq16RF09PL1er2sLxLWktvtlv3BfC+DqKio\nS3KlgPbiJk+eTFVVFX369JEXv9VqJSkpiUGDBsmN5nK5eO2112htbZWbceTIkUycOFG63r7v+1wK\nIvC+fPlyAD744ANaW1vJzMyU9RVGo5GioiLCw8OZO3cuAPfff39ADohOp8NisTBp0iSpYIiLpbq6\nWo6Q2LJlCzk5OcTGxl4268nr9fLII4/IC27+/PkBF5aiDkz0PCstLZUpyb7dK84fzFdbW8vixYs5\nfPgwoO39pUuXdjr43xEmk4nhw4fzr3/9C4Dnn39erv2uu+4CtBlRlZWVGI1G2V/Sn3tJXLCff/45\nK1euBDTXlGhCGx0d3c5zAZpVsnDhQpKSkmSatBBO/nhGer1eJkhFRkZKpfmH/rbD4WDdunWUlJTg\ncDhkxmVeXl7AMuQ8Hg+//OUvAS1bNDExkTVr1siyE9/1+gptsfd9mxIHKpHqihZQQDsz+fuwWCyk\npaXR3NzMRx99BGgP9+TJkyQnJ0vXw+TJkztVcBYaGsrgwYPxeDzSpwznNA7flPbW1lZ27NiB3W6X\nvzdq1Ciio6MDUsDocDj4/PPPpdszLi6O6dOnExsbKwX07t27SUlJYcyYMSxbtgzQtKhAudbOb80T\nEhKCx+MhKSlJamdWq5Xrr78+IDUWF4rD4aCmpkYeTBEzDBQ6nY4RI0YA2vd3OBwcP35cuq2EwiAK\nYIWlXlRUxNq1a9m6dasUYuPHj2f58uUB0cD1ej2DBw+Wvfg2btxIQ0MDeXl5MjstNDSUiIgI2tra\n/N7DzeVyyT6cDz/8MDqdjm7dupGbmyvdsUePHqW2thav1yvdeatWrWLo0KHt4rpiqKi/YiniefvW\nFXWkdIq7q7y8nDfeeIOmpiaioqKYP38+oHl/AnH2vF4vRUVFsoFwWFgYTz75pCzK7ej3fXsZivBA\noFt9qSw+hUKhUAQlV7wFBefMfIfDISet+lpWbrebyspK1q1bJzWulpYWnE4nY8aMoV+/foBWqNsZ\nU9VkMjFkyBB69+6Nx+OR2Xu+Fohw0/zzn/9k+/btOBwOxo4dC8D06dM7Zc7/mPaXlJTErFmzAMjJ\nySE9PZ2vvvpK+pz79Okjg7li7V053hnO+bpFQDYrK0s2071clJaW4na7pcu2K8aOCBdPZmYmxcXF\nFBQU8N577wFa52uRzHPy5EnpFaipqaG0tJSoqChmz54NwMqVKwNqffomjOTk5GC32+XIE9DmMYli\nTvH8/BXPPHPmjGzTVV5eTmhoKMnJyRQWFsokG6fTicFgoF+/frzxxhuA9mw76njhT3ybsXo8Hpqb\nm+W+8fWmiHU+8MADHDhwAIPBwPTp06UH40KmcF8sohXTX//6V+bMmQNoM55GjBjxnfPuO2NMZBaK\n+VAjR44MuNv9ihdQXq9XBtT37dtHQ0MDGRkZ7eIm7777LqtXr6a+vl66jiwWC+Hh4WRmZsrDHBER\n0WlXiGhJ5JvJJP633W6XJvXTTz9NU1MT3bt353e/+x3Q+YvvhzZLSEgIAwYMkBdHZGQkbW1tlJSU\nyJjbLbfcwtChQzEYDJelU4PA6/XKQmXfvoSXC9GiJiMjA+iaLhZCqKxdu5Z58+ZRUVHBs88+C2iX\ncV5eHrt27eKzzz6TLr5evXoxadIkpk6dKoWGv3s4doS41ETGpcvlanfOhIJWVFQE4JcRN16vl82b\nN8uCVtEmyOFw4HQ6pTsqKSmJu+++myVLlnTYFSXQZQt6vR6LxYLb7ZaZseLsFRcXyy4qu3btwmAw\nMHbsWP7yl7/IMxmIdbW2trJ7924GDBgg4789e/aUbjuxv+12O01NTVRXV9Pa2iq7Y0RHR3PjjTdK\nxSOQXPECSnQ+AC2GsnXrVoqLi9v1uWpra5MzYEQcIS0tjXnz5vGb3/ym3bgCfyDWIwSUx+PB7XZT\nV1fH7t275c9ycnJ4/PHHGTRoULt/FyiMRqOMA5SWlrJjxw5aWlpkfc+QIUOkALtcCQmgPQeREi3m\n+FzspFV/snv3bnQ6nQwedwViL2RlZfH++++zfv16qdy0tLRQVFTE0aNHSUhIkFN9lyxZQu/eveWs\nLOja9yjOYkhIiDx7grCwMLkmfwwI9Xq9HD9+XApC0ZWlubmZkJAQmdK9Zs0aBg4ceFkzQPV6PWFh\nYXKt9fX1bN68mTVr1shkFoCBAweycuXKgA0DFJ6mF154gQ0bNhAZGSk7XqSlpVFeXs7hw4dlXOnM\nmTMMHDiQbt26kZqayoIFCwBNQGVmZnbJebziBRScy2qaPn0627dvp66ujtbW1u8UkRmNRtnuZfXq\n1fTs2TOg7g/fWSk6nY7Y2FjuvPNOQMvOGThwIPHx8V1yeHQ6HSaTSab/lpaWYjQamTFjhqx7CIQ7\n4VLQ6XSMHDkS0J6d2WzG7XZ3ubsRtL2TmZlJYWGhTI8OVFFiR+j1etLS0rj33ntlDd+3335La2sr\n8+fPx2q1Sm3btyThclnA588Hstvt1NXVYTAYpAV14sSJTjdkBa2b/HPPPQdol75o0XXzzTfLZsNi\nMN/lxmAwyP1jMBhIS0sjJSVFln2kpKTw+OOPk5OTE7D1Crf5+++/zzfffIPD4ZANfCMiIjCbzaSm\npkrhHhcXR3Z2Nrm5ue3uqa5UFFWShEKhUCiCEl0gCjD9wCUtSgSNn332WV555RUZxwBNG1iwYAEP\nP/wwQKdbmlzq+oSZf36j1q76fGHSi0Dy1KlTZfD6clgoHeEbVxQjUcTsrMvhqjl79ixr1qyRscoe\nPXp0+d7xjWmK6coioeRytX/6IYQbb+/evSxbtoyTJ08yfvx4AGbOnClbjXUW4UqsqanBarUGtKjV\nX4hYz549e2TS1syZM0lJSQnoGRT7p76+nrlz53Lw4EF5R5pMJhISEvi///s/aakPGzbMXxPHL3lj\n/qQElPzH/78+RBwS0booEHNUriREogZoBzouLo7Q0NCgPNDi3fm6aX2zIf+X3+OVhNvtpri4WMYz\nQHMnnT+a/n8N0TpI7G/RHaSrFVbfOHkAk6OUgFL8dOnKmM+VgHoeiisMJaAUCoVCEZRcsoAKPt+O\nQqFQKBQoAaVQKBSKIEUJKIVCoVAEJcFaqKsiwAqFQvE/jrKgFAqFQhGUKAGlUCgUiqBECSiFQqFQ\nBCVKQCkUCoUiKFECSqFQKBRBiRJQCoVCoQhKlIBSKBQKRVCiBJRCoVAoghIloBQKhUIRlCgBpVAo\nFIqgRAkohUKhUAQlSkApFAqFIihRAkqhUCgUQYkSUAqFQqEISpSAUigUCkVQogSUQqFQKIISJaAU\nCoVCEZQoAaVQKBSKoEQJKIVCoVAEJUpAKRQKhSIoUQJKoVAoFEGJElAKhUKhCEqUgFIoFApFUKIE\nlEKhUCiCEiWgFAqFQhGUKAGlUCgUiqDk/wGU+cCUakDTdAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7c16ef128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_multi_images(outputs_val.reshape(-1, 28, 28), n_rows, n_cols)\n",
    "save_fig(\"generated_digits_plot\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interpolate digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./my_model_variational.ckpt\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo0AAAB9CAYAAADHoFl9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFe5JREFUeJzt3XmwjvX/x/GPioSicOyO7GTNWshQlqJ/UnZJy8hkGi0T\nM2pMmsI0woxRowaNMTJRoxpMthkpEiZrlsjuWIvIlvr98ZveXtfne18+x3Gfc+5z7ufjr5dzzr24\nPvd13Z+53p+lyL///usAAACA67klv98AAAAAUh+dRgAAAATRaQQAAEAQnUYAAAAE0WkEAABAEJ1G\nAAAABNFpBAAAQBCdRgAAAATRaQQAAEDQbfn0umxDk7uKJOl5aKfclYx2oo1yF22U+mij1Ecbpb5s\ntRF3GgEAABBEpxEAAABB+VWeLrD+/ffaHfIiRZJVBQYAAEht3GkEAABAEJ1GAAAABBWq8nROSsf6\nGM1ZWVmWixYtarl48eKWS5YsafmWW+h/AwCAwoueDgAAAILoNAIAACCoUJWnczKb+erVq5YvXLhg\n+eTJk5aXLVtm+a+//rI8YsQIy6VKlbrh18bN0yEF2fk5wwjyhh7/f/75x/KVK1csnzhxwvKtt95q\nuXTp0pHn0mEgyB16HdyzZ4/l9957z/KBAwcs33///ZHHDxs2zHLZsmUt+235H1aeAAomvkEBAAAQ\nRKcRAAAAQXQaAQAAEFSoxjTmxG23XTsEOt7tyy+/tLxixQrL/fr1s6zL7yC5/DGJly9ftrxjxw7L\nkyZNsrxx40bLOkYuMzPT8oMPPhh53kcffdRyvXr1LBcrViwnbzut6XjFbdu2WX711VctaxudPXvW\nsp57ZcqUiTzvxIkTLQ8cONCytjGyR88rHcO9fv16yx988IHlRYsWWb799tsTPtY5555++mnL1atX\nt8zYxeTRtjt06JBlPdfatGlj2T+PaAskA3caAQAAEESnEQAAAEFF4pYmyWX58qKJ6P9/w4YNlvv0\n6WNZlwYZPHiwZS2Napk7BSSrDpFv7aTlaOei5ZhRo0ZZ1qED+pgSJUokzP6QgipVqlgeOXKk5Q4d\nOljWslySJaOdUqaNdGmq559/3vKxY8cs6/kWt/yR//OMjAzL69ats1y5cuUbfMc5UqDbSJc7cs65\nixcvWtb2WrJkieXjx49b1iXGdGiHDhNwzrmKFSta1vMlj0qiBbqNskvb4vHHH7esQz6mTZtmuXfv\n3pHH5/NwjkLVRnpenT9/3vL+/fstjx8/3vLatWstazvod5Nz0aWvdLjU0KFDLbdt29aynmv6vDlc\nWi5bbcSdRgAAAATRaQQAAEBQStVU84OWy7QMrSUaLcP9/fffCX+eYuXpAknbQo+zc85lZWVZ1p0p\n9JZ8uXLlLNevX9+ytquWuf3fjRkzxvLHH3+c8LnSfUcZbaM///wz8rsZM2ZY1jJohQoVLHfp0sWy\nzlz/6quvLC9cuDDyvFre1tcYPXq0ZWaGXhM3Q9o555YvX25Zj3ONGjUsDx8+POHPixYtatk/D/T4\n0xbJo+VK/ezrsBzdjUzPFf8aymoDOecP49Nr35QpUyxPnz7dsn5naTsq/zzS19m+fbvl1atXW37h\nhRcsP/bYY5Z1lyY9V51L7jmZ3t+AAAAAyBY6jQAAAAhK+5qqzoLS2dN6O1lvIf/++++WDx8+bLlm\nzZqR56UUcHP8WZ+rVq2yfPToUct33nmnZZ3xXKtWLcs6o03LN85Fy6h79uyxrOXS2rVrW2bR72v8\nNqpWrZrlbt26WR4wYIDlzp07W9Zj2bFjR8vff/995Hl1SMGWLVtu4h2nBy1x7d69O/I7LXHqNUsX\n565UqZJlHXZzvRIXJencoUOgli5darlhw4aWe/XqZbl9+/aWaZPk8cvTp06dsrxy5UrL586ds6x9\ngLh8xx13RJ5XV/e4dOmSZT1XtV392deJ/ibZuNMIAACAIDqNAAAACEr78vTJkyct68wnvTWst4y1\nNKoL4g4ZMiTyvCVLlrRMmSB7tASgx9k55z766CPLp0+ftnz33Xdb/uOPPxI+b+PGjS1rOdq56MK4\n+vpxM+Z1Vlo6tqv+n/3SSt26dS1rqbpdu3aW4xZ+1uOte1I7F22X8uXL5+RtpxX9vPqlfp3RqaVM\nPY90OA6zovOeft7nzJljWfeY7t69u+VWrVpZ1u8qf/gIcs7fyGDq1KmWdZazHvO77rrLspaXu3bt\narlfv36R561atarlM2fOJHxeHTqnw7Pyqr250wgAAIAgOo0AAAAISrvy9JUrVyL/njBhgmUt3eis\nQZ1NWKdOHcu6MPS+ffsiz9ugQQPLzKTOHm2bcePGRX535MgRy1q+0dnsWrbWEl2LFi0sN2vWLPK8\nOvNNF0LWdtb3peXVuDJeuvD35H7ggQcsly1b1nLcDD8t+cybN8+y7rHrXPTY6mvgmrihFf5i9pmZ\nmZZ1MWC9RvkzRZG39Pjr8CmdJa3laZ3Jq1lXfcCN03bwV93Q1TV0oW+9JrZp08ay7h39yCOPWPZX\n49DvlLjSsw7f0b/XsrW/sHsyNx/hTiMAAACC6DQCAAAgKC3K03qbWRfkds65uXPnJnyMzv4cO3as\nZS3j7Nq1y/K6desij69evbplnUWFqLi9jPX2v3P/O6wgEZ0l3bt3b8takvNLbzrjVxcBP3/+fMLX\nYDbpNX7JQ49l3OxzPf4HDx60vGDBgtjXueeeeyzrIuDpfvyVHgud4a8zNZ2Lrhags3H1HNE93LVE\nxvHOe23btrWs5Ufdb1pX8ahXr55lhnLcHC0JT5s2LfI7/a7Qa5qumtK6dWvL2i7ah8jJOaWrVuhr\n6/X4enta3+x5zJ1GAAAABNFpBAAAQBCdRgAAAASlxZhGHZvw+eefR36nq67rmIC+ffta1rEhOu5O\nl4E5cOBA5Hl1dxLGBcXTthk9erTluN1dnIu2U48ePSyPHDnSckZGhmUd4+WPadT20F0xFEuQJOZ/\nlvU465hQXULn+PHjlnWckO6q4OvcubNldoQJ02U/OnToEPmdjl3Ua+Hs2bMt65IgOqbOHyeF3Kff\nHXrd27x5s2VddqxKlSqWWertxum1Xpdw++STTyJ/p+O09Tqox3zDhg0J/75nz56W/WWR/F22Ej1v\ndsbV+z9PZr+DqwAAAACC6DQCAAAgKC3K05cuXbI8Y8aMyO90KZcyZcpYbtSoUcKfazl7xYoVlv0d\nYSpXrmx58ODBlv1dNNLdxYsXLS9evNiyXxLW0swzzzxjedKkSZZ1uYO42/H+pu66jMWvv/6a8Lm0\nLEepOp62keYtW7ZYfv/99y2vWrXKsp6j/g4y/fv3t6wlcIT5yyLVqFHD8pNPPml5/vz5ln/88UfL\nzZs3t3yzS4Uge/SapOVpPf579uyxrDtZ6U5MtNGN0+v70qVLLeuwtOs9Rkvay5Yts6zXOv2eu+++\n+yLPVbNmTctNmza1rLs36XeTLueXVzuUcacRAAAAQXQaAQAAEFRoy9N6y3j9+vWW/TKy3sbV28Ht\n27e3rLd9N23aZDluBptzzk2ePNlyy5YtLWu5J13LB9o2u3fvtqwzpv2y2lNPPWV5ypQplv1SZoh/\nzLXdFi1aZLlChQqWdacZZpBmjw47+Prrry2vWbPGss6q1tJbkyZNIs/VqVMny+l6ziSLfn61VP3w\nww9b/uyzzywfOnTIcp06dSzTDrlHh2roOaIrdOgqBLpjkg6Loo1unH43VaxY0bK/q5uuDqEznnX4\nmQ6l0bbQmdT+qivHjh2zrOee9ju6detmWcvWedXefAMCAAAgiE4jAAAAggptefry5cuWJ06cmPDn\nzkVvLeuC3uXKlbOst5N1AWgtC5w8eTLyvFlZWZZnzpxpWUud6ToTVEsAn376acK/qVq1auTfb7/9\ntuW4BVCzw589vWDBAsu//PKL5YMHD1rWkkHcAuCInid6LL/99lvL586dS/hYnSXaokWLyO+YtZs7\n9LjWqlXLspZHdaanlrNZBSJ5/BUZtPS8Y8cOy1u3brV84sQJy4MGDbJcunTp3HiLaUOvLzqzefjw\n4ZG/0+FLOvxMnT171rKWt7Xf8PPPP0ces3DhQsv6XaXPpddQ/RvK0wAAAEgZdBoBAAAQVKjK03qb\nX2/76sKa/n6cus9qr169Ej6vlt301rDuy3r06NHIY3SW4qlTpyzrrNJ0LU/rMdAyps6Y7t27d+Qx\nWq6+0dvw+rnwy6Pbtm2zvH//fstaLvVL2vh/fllNF0d/7bXXLGsba9uVKlXKcsOGDS37QwC0XKpD\nE5jJnjx6LLX0vHfvXss6k7dYsWKRxzNsIHnihj3p/uANGjSwHDdLF9kTt2GDtsMbb7wR+Z1+/uOu\nQ9oW+hr6faILdTsXvfbpsB5dXFzPz/zYX5yrLgAAAILoNAIAACCoUJWndR9pnW2rM6b92WWvvPKK\n5eLFiyd8Xr0FXLt2bcu7du2y7JdrtKQdd2s6negxWL58ueWffvrJcqVKlSzrvrjORdtAnyuuHKN/\no+VwncnunHPfffedZZ0lrbPdtPwTV8pIl7KQ/v+PHDkS+Z0OKdi5c6dlLd907NjRsg4HuV7JRRfA\n1XNUS9XpcvyTSdtSF9bv169fwp/r+eGX1fzrH3JOh+noAs/6naLnkZ4TOvzG/z7jHLlGv4fjZiDr\ndctfseNGj6X+vWa/P6LfgbrxhK7mEfce8wp3GgEAABBEpxEAAABBhao8rbNf165da1lnX+qMTeei\nt/zPnDljWUuSus/kxo0bLX/zzTeWT58+HXneq1evWtYynr+ncrrQEvGoUaMsazlFS5R+aUWPZ9zP\n9TV0T+tZs2ZZ1sW8nYu2uZZF69atm/B96edF2zJdSj+62LDuV+xctHypZZOuXbtanjt3rmUt+cTt\nsZvo3/9p1KiR5XQ9r26GDufR8rLOzNVrp5bLtGztXHSmKW2RPLqg+ltvvWVZz6m4xcDbtWuXu2+u\ngNEytH7v6Oe6fPnylvU7KLeu7/7z6lA6XfVFh4NkZmbm+vu6Hu40AgAAIIhOIwAAAILoNAIAACCo\nUA0+0V1ZdOyZ1v315845N2HCBMutW7e2rBuS6/hIXS5GV+v36Ti47t27W86PFdzziy7poW1z+PBh\nyzr+SXfY0bE8Ph1PtWbNGstjxoxJ+Hq6mr6O4/Lpa+p4OR2vpWP10mUco47/eemllyzrWCrnop/5\nnj17Wp4+fbplHVOsx1LH7OhyR845t3XrVsvr16+3nJGRYVmXqkiXdsmJuOW/dAckHc+t56c+9sKF\nC5Hn1XHf2n60xY3TcW3lypWzfO+991rW8cD699u3b7fMmMYo/e5fuXKlZb2+DBgwwHK1atVy/T1p\n2znn3DvvvGM5KyvLcvPmzS3rMj2MaQQAAEBKotMIAACAoEJVntbyspaElyxZYtkvq2i5S3d4iZuS\nH7eji7+Uz8CBAy0PGTLEcjrtnKDlLF3qRstfukH7yJEjLV9vh51169ZZ1h19dMklfW19Ln8niypV\nqljWXRYGDx6c8P2mS7lNj7furqTH3vfQQw9Zfv/99y2XKFHCsraLLpekZRp/iZ0VK1ZY1lKStot+\ndljy5Rr/ehW3dJWeI/oZ1+E0WsLWoQjORYeM6HPlxbIlBZ3fRqdOnbJcvXp1y7obmdLzwN+5BInp\n0nk//PCDZb0+vf7665aTOaxM23vx4sWR3y1btsyynqvDhg2zrNfT/MCdRgAAAATRaQQAAEBQoarj\naClk2rRpll988UXLq1evjjzm7NmzlnWWrT/L+j9a+qpfv75lLcc551yHDh0s6+3kdCrRaMlRy5pa\nvqpXr17Cx2pbOOfchx9+aHnq1KmWT5w4YVlLC1o+a9OmjWUtQTsXnbHdpEkTyzpDLW7nF309zc7l\nz0byybRz507Lc+bMsaw7t+gxcs65qlWrWt67d69lHd6h7apDQHQnC91pyX8vevx1Nx/9rGkpqTCf\nb3GfP10hQHdJci5+ByQ9Tnru6LAbPcY6FMS56AoTjRs3ttyqVauEr4F4epy1LbSN9W/27dtnWduL\n4x2l12Q9ltu2bbOsM5afffZZy7o6g3M3fmy1JK07vWjZ2bno8Dmd/Z5KK7AU7G82AAAA5Ak6jQAA\nAAgqVOVppYuizp492/KiRYsifzdv3jzLWmLRsrUu7jxp0iTLXbp0sezPJszvW8ipQG/JHzt2zLIu\nBKxlzKFDhyb8uXPRxbr9GfD/0XKpLtL68ssvW65cuXLkMXElOv153ILIqqCXo52LztabP3++ZZ3N\nqcdCy8vORVcp0EXXdaiHtr0eSx2y4C94q+9L269Zs2aW02WGux4zPa6HDh2yvHnzZsv+eXT69GnL\nv/32m2UtW+u1TBe817/RVSecc27Tpk2WR4wYYVmHfDCzNzH/mnLw4EHL+lnW8003ltC2HzRokOXC\ncE1KJr2mV6xY0bLO/Ndr3cSJEy2/++67kefyv+8T0evYlClTLI8bN86yfw3VMrgOsfNXZ8lPfKoA\nAAAQRKcRAAAAQUX8WZ95JF9e1Ln/neWqpYG4hW/1Nn8BmZmZrDd2U+2k5bPJkydbHjt2rGV/dmcc\nncGsZa6+fftaHj16tGUdnhA3+zk3ZfN1kvFmknYu6bnwxRdfWB4+fLhlLYv55S89N+LOk7iFn+vW\nrWtZ284555544gnLOotQF5vOxQW9U6qN9Pp16dIly1qi3L17t2VduNi56N7EWrrWNipbtqxlnS2v\nQwv0tf3HTJgwwXLLli0tZ6ekl0Mp1UY3Sme7O+fcrFmzLOuGE1ru1HNVzw9dtSPFFrlPqTbSVTf0\n2qNDMPT61qlTp8jj4xb+1nNq5syZlnUxcf1e9Bfqnjt3ruUePXpYzqPvrWy9CHcaAQAAEESnEQAA\nAEFpV55OEylRntayi84IHDVqlGXdS1j3odaZ6c4599xzz1nOyMiwHLdnbm6JW1DZf+2CWJ5W2naH\nDx+2rPtCa7nSuWhbxM1m1lmLWl7W7O87nh/DC0TKtpGWKDXr59LfpECPX9zjla5UoG3qt5H+Ox+G\n8KRsG2WHf+x1Jvr48eMt67nXv39/yzpER4d2pNjwqZRqIz1HdP/nPn36WNbrm99P0mMbt+FD3PHX\n1ViWLl0a+V3Tpk2Dj89FlKcBAACQHHQaAQAAEER5unBKifJ07JNm4xZ+mkipkg0Soo1SX6FqIx0a\noptMHD9+3HJmZqZlXYUghRf0Ttk20u8j3YRCV+PQErZz0VnW2l76XLpQ95tvvmlZF2BPsQXvKU8D\nAAAgOeg0AgAAIIhOIwAAAIIY01g4pfSYRpiUHecDQxulPtoo9dFGqY8xjQAAAEgOOo0AAAAIotMI\nAACAIDqNAAAACKLTCAAAgCA6jQAAAAii0wgAAIAgOo0AAAAIyq/FvQEAAFCAcKcRAAAAQXQaAQAA\nEESnEQAAAEF0GgEAABBEpxEAAABBdBoBAAAQRKcRAAAAQXQaAQAAEESnEQAAAEF0GgEAABBEpxEA\nAABBdBoBAAAQRKcRAAAAQXQaAQAAEESnEQAAAEF0GgEAABBEpxEAAABBdBoBAAAQRKcRAAAAQXQa\nAQAAEESnEQAAAEF0GgEAABBEpxEAAABBdBoBAAAQRKcRAAAAQXQaAQAAEPR/XDgv8APvm50AAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7b1671dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo0AAAB9CAYAAADHoFl9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFklJREFUeJzt3XmwjvX/x/F3e5FSZM2WSJZkK20yiZoWKrQNZZoQozLT\nNowZmjRt32ow/cGMlGYqlT1FESpMOpbsa+IQsku2tt9fv7fX5+pcPudwjrPcz8dfr6/7Pte5uz7n\nuu7P93p/ltP+/fdfAwAAAI7n9ML+AAAAACj66DQCAAAgik4jAAAAoug0AgAAIIpOIwAAAKLoNAIA\nACCKTiMAAACi6DQCAAAgik4jAAAAos4spN/LNjQF67R8Og7tVLDyo51oo4JFGxV9tFHRRxsVfblq\nI540AgAAIIpOIwAAAKLoNAIAACCKTiMAAACi6DQCAAAgqrBmTxeIf/89NrnqtNPyPllLf/7QoUOe\nzzzz2Gn6888/PZ977rmezzjjjDz/PgAAgOKCJ40AAACIotMIAACAqGJTns5N6Vn/Pbel6qNHj3o+\nfPiw55UrV3qePn2654ULF3q+//77PXfq1Ck4LuXqgqNt+/fff3v+559/POswAn2/DikwC4ceIP/o\nOde22LJli+cxY8Z43rlzp+dkGw0YMCD1NeQPba8jR454zs7O9lyqVCnPF198cfDz2i4nMjQIQPHA\nk0YAAABE0WkEAABAFJ1GAAAARBWbAV15HSeT2/efddZZnnfv3u15xIgRnufPn++5QYMGnlu0aHHC\nnw/Hp2OszMJxVnPmzPE8ePBgz8uWLcvx/eeff77nxo0bB8e99957PXft2tXzeeeddyIfO6Pp+NLt\n27d7HjRokOfJkyd73rZtm+fTTz/2/1/LlCkTHFf/d9++fT2fc845J/eBM5yONdV73Isvvuh55syZ\nnlu3bu1Zrzszs2bNmnlmnHDB02tN21HPfXJcPd9RBU+/t3SMvbaX3uuKYxvxpBEAAABRdBoBAAAQ\nlfF1hL/++suzlj2/++47z/v27fNctWpVz/poWR9Fm4WPoJE7+mj/4MGDwWs//vij5+eee87zmjVr\nPOuj/bPPPtuzlqq1DGdm9tNPP3nWkvRDDz3kmXJbzrQsZhaey8cff9zz6tWrPet1ceGFF+b478my\n8/fff++5Z8+eqe9DnN6ndHiADsdZvny557p163pu376953r16gXHLQ5lteJIr7G5c+d67tGjh+e9\ne/d61jbq379/cKxq1ap55p6WN8nhUtouP//8s2ddRuzjjz/2rNdH8+bNPXfs2DE4btOmTT2XK1fO\nc9qSVoVx3dGzAQAAQBSdRgAAAERl3DPq5GNmndW0ceNGz1oe1UfA5cuX97xixQrPOkPXzOyiiy7y\nTKk6d7QttKRpZjZq1CjPOjNX26NRo0aer7/+es+LFi3yPHv27OC4+/fv9zx69GjP9913n2dKOcfo\n9bNp06bgtZdfftnzjh07PLdp08Zzv379PGuJU0uir7/+enDc9evXe96zZ4/nsmXL5umzZ6Lk/e7Q\noUOeJ06c6PmPP/7wrKXP7t27e9ZZ7LrqhBnl6fykQwj0uujVq5dnHZaju/Ns3brVc/Jep7uWlS5d\n2jNtF5e8jnTImn43vf/++5537drlWc+xttEPP/wQHFfvie3atfOsq3xUqFDBsw6RO1XtSG8GAAAA\nUXQaAQAAEJXxdbfDhw97XrVqlWedHaXlZX3kPGHCBM9aqjYLZ3kmS9fImZanf//99+C1devWedZy\nsT7O79Onj+fatWt71hmgOgMx+Tt1Fpy2s5ZyMp2WzrTsbxaW+rUk/eqrr3quWLGiZ72udKH8ypUr\nB8ddsmSJ519//dVzrVq18vTZM1FyVQctdy5dutSzlsJ0Bq6WPrW9kuU6Spz5R1f0+PLLLz1nZ2d7\n1pnQWrY+3hACvdchb5LX0W+//eY5KyvL84EDB3L8eb0+jh496lm/Z8zCYQf6WqVKlTy3bdvWs343\nJRcKLyg8aQQAAEAUnUYAAABEZVx5OvmIfuTIkZ4/++wzz/qYWcvLOntTywg6+9DMbMOGDZ51v2pm\nUqfTR/hakjQLy2paKtAFnnVBbz3PWmJLlmz0fbqAalopjjLcMcm/ZT23DRs29JxW4lR6vX399dfB\na3rN1qlT58Q+bIZKLsCuCw7rNda5c2fPeh3oude//WQ7cl3kHy196koCuqKHbj6ge4LrrN5kuTJ5\n70PuJa+jDz74wLNeR7qRhF4jutKDDi3QYTlmZjVr1vSs90RdJUT7HXp96vdUQe5pTQ8GAAAAUXQa\nAQAAEJUR5enjLUj8v//9z7Pu4Zm2R66W3fRRdHIB6N27d3vWR8iUp9PpY/fx48cHr+mjei2zrF27\n1rPuF66zr/Wc61ABs3Cx45YtW3ouVapUnj57ptAyR3JxbS1r6v6q+jN6Lep1oXu2JmcUNm7c2LMu\nmo+4ZFlN71P169f3rCUzvQ51KIjOzOU+ln+SM9F1KI4Oe9JhHnfddZdn/X7S77Dk6gI6lIfhBHHa\nLjoszcxsxowZnnXVCD2vl1xyiWddWL1bt26ea9SoERxX20hnWacNydLrUH93crZ32vtOBFc+AAAA\noug0AgAAICojytNaBhs8eHDwms5U08fRWmrTvVh1T+OdO3d61kU5zcLHw8nyA3Km5ym597S2oZ5b\n3Yda20NLNlWqVPF8++23B8fV91155ZWeT9VCqcWNlja0bGxmdvfdd3vWhdL1fVr214W6dR9kLYOa\nmQ0cONAz+4Dnjd7HzMJZ0sOGDfOs++HqkI2qVat61ranvFlwNm/e7FlXhLjiiis8a7toSVrfr3sU\nmzGkIK/0+0iHz5iFQwh0CIiWjhs1auRZhz7pUJDzzjsvOK5+72jWz6L3wLQ2Lcjrk78iAAAARNFp\nBAAAQBSdRgAAAERlxAAhHfMxderU4DUdH6fjAG666SbPDz/8sGcdb6XLISTHWjF+JO+2bNniWZcx\nOB7dradVq1aedWkkbZvrrrsu+PmNGzd61vF2tF/O9BrRJUDMwjGNX3zxhWcd76tjU7/66ivPOoY1\nuVTItddeexKfOLMldwGpXbu2Zx1/pfdFHdurY+dwauiYUr2/6XJhuhyLjn0rV66cZx3fiLw7fPiw\n56FDhwavaRspvd70XqfLwek4Rv2eMguXFNPrU49b2GOL+WYEAABAFJ1GAAAARJXY8rROUdcNxZMr\nuyt9NNy7d2/PlStX9qyPg/XxdVZWVnAs3S1Gl0qoWLFijsdCuIuLPpo3Mzt48KDnCy64wPM999zj\n+dZbb/VcunRpz1om0CEFZuHyFjpUIT9X0C+pkudFd0Bo166d5xUrVngeN26cZy2JailNh4OYhcMG\naIuTo0tM6TIg2ka6XJIO7UkOR0D+SC7Jlp2dneNrujzcrFmzPGt5Wu+ByV1BkDe6DJWe++PR3ZRW\nrVrlWZfomTNnjmcdTmBmdvXVV3vu0KGD5yZNmngu7PshTxoBAAAQRacRAAAAUSW2PK2P5j/88EPP\nunq7WVgW69Kli2d9zK+zb7VcoDOoFi9eHBx30aJFnnXm06BBg3L8d5hVr17dc506dYLXduzY4fnB\nBx/0/NRTT3nWmYZaXta/BT2OWbgrie6ewS4+eaelEi277Nq1y/O0adM8b9u2zfMNN9zguWPHjsFx\n2Z0n/+i57NSpk+f+/ft7njlzpufLLrvMc40aNXI8Dk5OsoyswwP0tQULFnjW+5gOC9F74KWXXhoc\nN7n7CP5L7/ubNm3ynFxNQ/sE+r2hbaErFWg77t6927PeA83MPv/8c89aEn/hhRc816tXL/JfUbB4\n0ggAAIAoOo0AAACIKrHlaZ0lPWXKFM/JUoDOZu7Vq5dnnb2rZTd9fK3/vm7duuC4WpLT39+3b1/P\n+igboaZNmwb/u2zZsp579uzpWWe8py3IrW2mpR8zs7Vr13rWWdk6Cw55p+dcZwvqjESd1a5DE1iU\n+NTQFQZatGjhWe9XK1eu9KzXBG2Uf5IrOuzbt8+zfkfoQtA6/EO/h+bNm+e5SpUqwXF18X2GF+RM\n+weak4twly9f3nP79u0933LLLZ51pQI933oPHD58eHDciRMnetY+TNomJIWBJ40AAACIotMIAACA\nqBJVntaSmO4VuXPnTs/JPaK7devmWWcKppWktUSjZQXdCzT5ew4cOOBZy6P6iLuwHzkXFn3s/sMP\nP3hO7u157733etbzpo/909pJZ0jrfsdmYbvpEAOduaZlhkxtp9zQ868L244ZMybH9+ui+TpbXhd5\nNwuHJnD+848O59CZ0TrLdsaMGZ51aE1yJi7tcuKS9zqddVu/fn3PdevW9azfVXrf1O+XSZMmBce9\n8847PVOezpkOmalWrZrnIUOGBO+7/PLLPZcpU8aznte0a0KHhVxzzTXBazpDXldX0T5EYeNJIwAA\nAKLoNAIAACCqRJWn9RHusGHDPGvZTB85m4WLQ6eVOjXr4uBawjze434tPyRLb5lIz+enn37qWRcw\n1VnRZmZ33HGHZy1paxlaj6tl5wkTJnhOznLXttH21FmjWi6irHNMcgH0DRs2eH7iiSc8axv16NHD\ns+4brsdKLnirex7r4sWURE+OluK2b9/uWRdXz8rK8jx+/HjPOqzHjIWj80r/3g8fPhy8VqlSJc+6\nisRVV13lWYds6GLRY8eO9awreJiF12GmbyyRNpRJz5kOMdNzb5a+ukpu6HdIhQoVgtf0/rZ582bP\na9as8Xzttdfm6fflN540AgAAIIpOIwAAAKKKfXlaHzO/9tprnvfv3+9ZHx83b948+Hktq2jpWUs3\nBw8e9Lxw4ULPWgLfsmVLcFwtOehCuMn9QDPRihUrPHft2tWznvPkbHQtp+hrOiRBF8WdO3eu5y+/\n/NKz/l2Ype8xraVq/Vw64zQTy6N6vlavXh28pjPcdW/cPn36eO7Xr59nPZfadjo0wCz8e2ncuLFn\n3fMVuaPtp6U4nY174403ep48ebJnXXlA329m1rZtW89pi+zjGL2n7N27N3itdevWnhs1auQ5bRUH\n3bddz31yg4LkxhaZ7MiRI55nzZrlWe81ej9LlvPz696fvIdpX0P7FGnfQYWBqxsAAABRdBoBAAAQ\nRacRAAAAUSVqTKNu9q3/ruMGdEyUWbhbiO7wosuHfPLJJ57nzJnjWXeaSY4f0TEPuuq7bkCfSXRM\nxm233eZZx5HqEge6ZItZOM5H2/aXX37x/Oabb3rW9tMN4o83rkdX9k8bT6KfMVPGNOr51h0ndMyP\nWbh8kY5j1KWUdKkKpeO1qlevHrz23Xffea5YsWKO7yvscT7Fhf79a7vWqlXLsy770aFDB8+688iy\nZcuC47Zq1cozy+/E6X1Px7GZheNF9bpIWxJOl6TSMdu6LI/Zf7+jMo2es/Xr13sePHiwZ91tTJcE\ny897vX4OXUrHzGzp0qWe9btGd6ApbNxpAQAAEEWnEQAAAFHFvjyttESiO39oSWzx4sXBz+hyB9nZ\n2Z5nzpzpWcvQWt7RR9bJqfMtWrTwPGjQIM9aMsiU8qZZWC7WEoouZTBw4EDPzzzzTPDzWprRNhs+\nfLjnGTNmeNZlebT9k+XRUqVKedbytJbo9uzZ41nLF/rZS1pbpi3NoqVmXZbIzOyxxx7z3LdvX8+6\n5FTa79Dzlyw1awln3rx5nl955RXPWsbDMcmSpJZFL7jgAs96Xej51+E0jzzyiGe9V5qF15veC0va\ndXEy9O9dh0Ill2tr2LCh57RhF3pe9TtJ71XJZcu03JnpdKk2vY/p37V+Z9WoUeOkfp+2vX7/vfvu\nu8H7dKk+3b2uWbNmngv7muJJIwAAAKLoNAIAACCq2D+v1sf3PXv29Ky7gOgj+2+++Sb4eX2Er6vE\naxlH6SN+3V3m2WefDd7Xpk0bz1r21DJrSaezjseOHes5bYcc3R0mScsGvXv39qwza9NK0rqrQoMG\nDYLj6sxD3TxeyxH6t5A2+7SwSwb5TctnOmxg2rRpnmvWrBn8TNWqVT1v377ds5Zc0srQOgRk1KhR\nwXG//fZbzzozV2fF6zWW6TOp9brTe5pZ2K46NCNtVQE9lu4AlJz1qfc7LXvjGD3H+vee/HtN26Uq\n7T0LFizwrKXPKlWqBD+TtnJBJtJVGPRvXNtl6NChnps0aRL8vA4zS7vfaBvpkKoHHnjAc1ZWVvAz\nek2OGDEix38vbJl9dwUAAECu0GkEAABAVLEvTystPY4cOdJzv379PGuJxSy9xKgzY3Vm5ksvveS5\nS5cunpML2uoj67yWLktK2VPLutOnT/esZWQtXT7//POedaFoM7O5c+d63rdvn2ct+Wgb6GLg3bt3\n96xDCszCdtbPpcdNzoz/f9pOyZJScWw3nWk7depUzx999JFnnXWoC30n3zd58uQcf0bLZ2nnW99v\nFi5+XKdOHc8667Ru3br/+e/JJHr+9HwtX748eJ8uHqzlSh1qoOXlcePGedZyWfI60vKbDlPAMXpP\n0DbShabNwlJo6dKlPet3il4jupC+tmn//v2D42bS0Kic6PnXv1H9W9b7lq6g0rFjx+BYTz75pGft\nd+h1OGnSJM9vv/225927d+f4mczMOnfu7Llly5aei9KQm6LzSQAAAFBk0WkEAABAVIkqT+vj95tv\nvtmzlkaTpYCNGzd61lKlLrCqC9zqzN+CemR8vNJmcSpd6/nUhdd1D1udFb1p06Y8HzdtEXV9tK/l\n5WSb6TlMm7WY20WoSxI9ZzrDXGfjJstdK1euzPE1LXvr6gNp++cmZ2VrGz/99NOedb/kktwWJ0MX\nQzcLy2Q6U1Rng+o9Tu+P9erV86z78pqF+yUX9ftSYdHzokOekve9YcOGedZhNrrI/nvvvef5l19+\n8azt0qlTp9Tfn+n0O2TIkCGe9Vzq0A79zjIzmz9/vue07wcdfqMztPUeqP0UM7O33nrLc9qwqMLG\nnRYAAABRdBoBAAAQdVpuFhItAIXySzNIftUhTqqd0mZ0vvPOO55nz57tWWdb64LcZmYPPfSQ5/r1\n63tOKz0XVCkm7Xo5wd+XHx+yQK4lLa0cbz9bpSUfbXstaet50pJNuXLlPCdXIkjb4/sUlduKbBul\nOd5M9HXr1nnWoTpahtZF7rVcp4sS69AAs/T9xU+RYtdGeh2NHj06eO2NN97wrEMI9DqqVKmSZ92X\nWGe4F7H92ItsG+k9XTclGDBggOcpU6YEP6MreOj3lh5LrwndfODRRx/1rBsnmIWz5QtBrtqIJ40A\nAACIotMIAACAKMrTJVORKE8jqsiWbOBoo6KvWLeRDt0xCxeYXrJkiWcdatCjRw/PtWvX9lxUZ9xa\nMWwjHeahe7abhXuv69COrVu3etaZ7zqc43ireRQyytMAAADIH3QaAQAAEEWnEQAAAFGMaSyZGNNY\nPBS7cT4ZiDYq+mijoo82KvoY0wgAAID8QacRAAAAUXQaAQAAEEWnEQAAAFF0GgEAABBFpxEAAABR\ndBoBAAAQRacRAAAAUYW1uDcAAACKEZ40AgAAIIpOIwAAAKLoNAIAACCKTiMAAACi6DQCAAAgik4j\nAAAAoug0AgAAIIpOIwAAAKLoNAIAACCKTiMAAACi6DQCAAAgik4jAAAAoug0AgAAIIpOIwAAAKLo\nNAIAACCKTiMAAACi6DQCAAAgik4jAAAAoug0AgAAIIpOIwAAAKLoNAIAACCKTiMAAACi6DQCAAAg\nik4jAAAAoug0AgAAIIpOIwAAAKL+Dw7VXfPWdetKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7c0081208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo0AAAB9CAYAAADHoFl9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFjJJREFUeJzt3WeMVVXbxvGFHaWDIh3pQ1FBWigyCiiIBMUQAcXKB1tQ\nowhGDSEKRIyCCgQhFkQCBkEkotIUMIA0JfSidBAG6SDFwvuBvDfX2s9s1gxzxjkz5//7dMmcs+c8\nZ83eez37XqXQ2bNnHQAAAHAhl+T1BwAAAEDyo9MIAACAIDqNAAAACKLTCAAAgCA6jQAAAAii0wgA\nAIAgOo0AAAAIotMIAACAIDqNAAAACLosj34v29DkrkIJOg7tlLsS0U60Ue6ijZIfbZT8aKPkl6U2\n4kkjAAAAgug0AgAAIIhOIwAAAILoNAIAACCITiMAAACC8mr2dNI7e/b8RK1ChQoF/x0AAKAg40kj\nAAAAgug0AgAAICjly9P//POP5X///dfy4cOHLW/fvt3yN998Y7lChQqWu3fv7h336quvtkwZO+d0\nWIDmv/76K9OckZFhWb//0qVLe8ctWrRopq9D4pw5c8byqlWrLI8YMcLynj17LB89etR7/4cffmi5\nRo0alq+88sqEfk6cE3eu6fnBuQKco/2Gv//+2/LJkyct796923KlSpUsFylSxDtWfjiveNIIAACA\nIDqNAAAACKLTCAAAgKCUH9OoY3bWr19veeDAgZZXrlxpuVixYpZbtmxp+dZbb/WOW6VKFcuXX365\n5fwwZiEZaLs4549XXLhwoeVnn33W8pYtWyz/+eefli+55Pz/N4qOaXzyySctv/zyy5YZL5d9OrZH\nxyV++umnlseOHWt57dq1mR6nZMmS3n9re+uYRuSMnmOHDh2yPGDAAMs///yz5RdeeMHyXXfd5R3r\nqquuyo2PCOS6i1lGT99z6tQpy8uWLbOs8x/0uE8//bTla665xjtufugf8KQRAAAAQXQaAQAAEJRy\n5WktoTnnP1qeOnWq5eXLl1vWUucVV1yR6XvnzJnjHbddu3aWq1WrZvnSSy+9mI+dEuIe+Tvn3OzZ\nsy336dPHsi5loG0b9z1rGc4550aOHGm5U6dOlhs3bmw5P5QM8oIuV+Wcc/v27bOsJc758+db1jJm\nWlqaZV2iqmnTpt5x9XU61AA5o8uDzJw50/Lo0aMtX3bZ+VuEth3tkHv0Oqj3Hh0qMHjwYMs7duyw\nXLZsWcv169f3jlu1alXLen+qVauWZR2WkyrXvYv536nXvs2bN1seM2aM5cWLF1vWa5rep6LDsPID\nznwAAAAE0WkEAABAUMqVp6OPg48dO2Z527ZtlrX8UqpUKcuNGjWyfOTIEcuTJk3yjqszrgcNGmRZ\nZ++myuP/rNK2Wbdunfez4cOHW9Y203LMbbfdZllnd86bN8/ylClTvOOeOHHC8gcffGC5YcOGlrVE\nl+q0jXQXF+ecGzp0qGXd+eX++++33Lt3b8t6Lmg7RHeE0fOE4R05o+23f/9+yx9//LFlLVv379/f\ncp06dTI9TvS/ua5lT/S71HNh1KhRlocNG2ZZh4IoLZVqOds5f6bu6tWrLesKFPXq1bOs171Ub9No\nG50+fdry3LlzLf/www+W9Tq2YcMGyzpcp0uXLt5xdYeyZB0CkpyfCgAAAEmFTiMAAACCUq7uFp09\nrQtC79y507I+jq9QoYLlwoULW9byzpo1a7zj/vbbb5ZbtGhhuWfPnpZ10W/4M9KipU8t2eiG7/fd\nd5/lJ554wrI+5m/VqpXlBQsWeMfVcs6KFSssR/9OcI620YwZM7yfLV261HKzZs0s66LQceWXCy2m\nnuqlsUTSv2tdiFgXz9eZuQ8//LBlbTvaJHGipU+9r+jQGh0OpTPZdfiUru5RvHhx77h169a1fMMN\nN2T6fmSNzmrXoWg6dEpL2Lt27bL83nvvWd66dat3XB0qoO2XTOcbTxoBAAAQRKcRAAAAQSlRntbH\n/ydPnvR+9tprr1nWBb3jypMVK1a0rAsSR2c66aNp3WNXZyYyO82n34GWo53zZwtqmax169aZ/rvO\nstW2jC7urT8rUaLExXzslKLl6b1793o/07/52rVrW45bFFrPS22H48ePe8fV4SHJOqMwv9Dz6vXX\nX7d87bXXWtbhHNpG0cXcFdevixe91+hGETrkQ+8dOkSnc+fOlnVBbz0Hoz/Te48Ok+L8yly0jb76\n6ivLuse0lq31nNBzR/sgOozNOX+YW5MmTSzrsIO8Ptf4CwEAAEAQnUYAAAAEpUR5Wh8tf/HFF97P\nlixZYln3O9bH9Po4uGTJkpavu+46yzpryjnnMjIyMn2dlhgQL7q4ty6UWq5cOcvaTjoDVNvyp59+\nsqwl1Oj7dcYvi0iH6fftnL+ygM7UVHouatbyjZ5jzvmlGeSMlqd11mz37t0t6/mle7trCVuHgiBn\noveEWbNmWdb20hUG2rRpY1lnuOusaB0+5Vz2h0Ppay60R3Jel0v/C9H7+8iRIy0fPnw40/cUK1bM\n8u233265evXqlnWYQfQ9en3VIQR5/X3zpBEAAABBdBoBAAAQlBLl6TNnzlh+6623vJ/pTCZ9BK+P\n8tu2bWtZF+cuU6aM5eg+n5s2bbLctGlTy5Q942mpuGrVqt7PtDSjC39/++23lrWspqVPXcBb9191\nzi+ztW/f3nJelwCSlZZJdNamc345ZdGiRZZr1KhhOW7BWm37aBvRFomjf+96jumqEDp7fePGjZb1\nescs28SJlj4XLlxoWc8pHU7QuHFjy9qmOpQjeq+JO4+yW6pOFdof0NnSzvnDp3RojV4ftSTdo0cP\ny3p9i5b99b51oSEBeYkzHwAAAEF0GgEAABBEpxEAAABBBXZMo44H+PLLLy1v2bIl9nU6bqNhw4aW\n+/fvb/n666+3rON6oktQlC1b1rKOF9IxD/Dp95+enu79THdr0d1hdMcEHTun3//69est6+4k0ffo\nGK9UHMOTFfo3r2N1nfOXZPnss88sz50717IuCaLfvY7ziS6xQ1skjo5zi1syTHf60fOlSJEiufzp\nUofed1auXOn97MCBA5m+R69peo7oskj67zoGNfozxtaH6XjS0aNHez/TeRJKzxFddky/e901Ru9l\nzjl35MgRy9WqVcvmJ/5v8KQRAAAAQXQaAQAAEFRgy9M6DX7ixImZ/rtzfulLN3gfP3685biyZdwS\nPc45t2vXLsu6YrwuCxN9T6rT77Z8+fLez3T5Al3+QDd8nzBhgmVdWV9LANFlDLREpzuRUBINiy67\noqXnXr16WV68eLHl5cuXW9a20OEglM5yj363uoSLljh1SSu99mmpmvMjcSZPnuz9t+4Qo+eItpHu\nSKI7v+iQkbS0NO+4ukSWDiXRvwna9Ty9b2/YsCH2dfqd6dJwWqqeN2+e5W3btsUeV6+hWt6uV69e\n1j70f4AnjQAAAAii0wgAAICgAlsf1Z1e1qxZE/s6fUyv5U3dVDwrux9EN52fNm2aZZ1pVadOHcs1\na9YMHjdVRWfQduvWzfL27dst624v+thf219nrGtJzjnnmjdvbrlw4cIX/4HhnSeVK1e2rGU1PS/0\n32+88UbLlMhyj5YidUjAu+++a/nQoUOW9Xxh2EDi6M4fOkvXOf/aF3fv0dKpDr/Ra6Cu2uGcv0JB\n165dLdOu5+lwgPnz51s+ceKE9zq9Run32rp1a8s6REqHxenucdGZ8npN1HNy1KhRlvO6vXjSCAAA\ngCA6jQAAAAgqUOVpfbT8/fffW96/f7/l6KPdLl26WNbZSlkpSSudLe2ccz/++KNlnXX49ddfW+7T\np49lZlL7tHzjnF8O0LK+Lkq8detWy/q3oG0ZXZRVy9vRmfW4eHqeablt3bp1lnUhWy3/6Cx2JJae\nR7qSg37/O3futKwlzuxeExFPr0/Fixf3fqareOis2TvvvNPy6dOnLes5tWzZMstHjx71jrtgwQLL\nnTp1shzd8CCV6T1g6tSplqOrbmhJ+sEHH7Q8aNAgyzqrXY/boEEDy/369fOOq8MLtF11+JvO0M4L\nXAUAAAAQRKcRAAAAQQWqJqqPgIcPH25ZH+XrbGnnnOvbt6/l6IzdzOhjap31Nn369Nj36Exe3QdZ\nPy/lad8ff/zh/bfuX6wzznQfT/0OtQynpe7oXp+LFi2yrLPaWrVqZZmyXPbpebJq1SrLWlrREo/O\nBtX93Z1jv/ZE0vNCF7bX802HfMycOdNyhw4dLEdLmsx4zx69JmnZ2Tnnbr75ZssdO3a0XKJECct6\nTdL2mjVrlmWdfeucf7+JztjGObpygF63ihYt6r2uZ8+elocOHWo5bgF8vR6mp6dbrlGjhndcnT2t\nbax9GL1u5gXuhgAAAAii0wgAAICgAlUT/eWXXyxr2VEf8+qsMeecq1SpkuW4Eos+WtaygpZ0Vq5c\n6b3n1KlTlrUsoOXU6IysVKTfgc5+fv75573XaVlfZ9dqubNJkyaWdajBkiVLLB8/ftw7ru5dPXr0\naMu6hyuzC7NG21KHEKxevdrySy+9ZFn3WdXZu1qqds650qVLW2aoQOIcPHjQsp5Hus+7zqTWPcTb\ntm3rHYshBGF6fug94aabbvJep7OpdcOBuL99fb0u2F6+fHnvdToDV89PPb9ScZiBfi96fcrIyLDc\nsmVL7z1DhgyxrG2Ule9Ph1Fp2TlKhx0k03ACrsAAAAAIotMIAACAoHxfntbHtg899FCm/66Pj+++\n+27v/frIX0sG+shaHyGvXbvW8siRIy3PmzfPO66WQbW8mZaWlun/Di1dpFKJQBdQfeSRRyxred85\n58qUKWO5TZs2lnWhb93XWxdW1fLNwoULveNqmWbTpk2WtTSRlSEM8IdejBs3zrK2lw4P0b95/b61\nJOqcv1C4Dk2gLbJPr3G6kPBzzz1nWWfvjhkzxvKIESMyfY1z/zvjHefElaT1/hJdzD5uBm4cvYfp\njPiqVat6r9Nr3a+//mq5Vq1awd9R0MTtMT1t2jTL+t03a9bMe7/eX7J7HdK+wZYtW7yf6eYT+jcS\nnb2dl3jSCAAAgCA6jQAAAAii0wgAAICgfD+mUXdb2bFjR6avqVatmmXd6cO5+CV0VqxYYfn999+3\nvGbNGsu6evyFpsRXqVLFcosWLSyn6i4w2k46DlXHxOmYEeece/zxxy336NHDsi6tc/ToUcu6jMS9\n995rWZcxcM7faSZuZ4VHH33Uso6vS0U6FkjbyznnNm/ebFmXEWnevLnluF2XdCxWdCkqHeNYpEiR\n4LFwXvS71DFTev3RMVt67nTr1s3yY489Zjm6Y1PZsmUtM9b0PL0v6PVFx6vp9+1c9r8/fb22qd4b\nnfN3OOncuXO2fkdBo+NLBw4caFnH0uvucXfccUfsseLmI8TtHjd27FjLe/fu9Y6lfxeNGjWynEx9\nBZ40AgAAIIhOIwAAAIKS55nnRdq3b59lXVpH9erVy3K0pKXT33W6/eDBgy3rI2R95KxLHUR/t25E\nrmUdLQPp+wt6SUeHAfTv39+ylgNKlChhWTeEd865Z555xrJu2K67h+zZs8fy9u3bLWvZ9MiRI7Gf\nS/+WtGytyzRpGTVVdifRv3kteS1YsMB7XbFixSzrkixZWUJEdxTRvwPn/F17Nm7caLlevXqWU6Ut\nskvLcM75Qzh0R55y5cpZ1iEYtWvXtqxl1N9//907rrZFqtPzRe8v48ePt5yenm45Wp7Oye/Te5Uu\nJeOcf+3Tti/o956QChUqWNZhaXqti36X9evXt6y7Ken9RIcjTJw40fKbb75pOTrER5fW6dq1q2XK\n0wAAAMhX6DQCAAAgKHmeeWZD3ON/nVmpj4m1pDVhwgTvWDNnzrSsu7poGUd/n5bRdBNznQXsnF9+\n0N0StDyelbJAdPZjfi0laDtpCUBnSTdo0MBy7969vffrrjyrV6+2PHv2bMvr16+3rGVvLf9Ed5rR\nUqh+Fi136uxdnVFXkOn5o2WtSZMmWdY2dc65ihUrWtbhGjpjV89RLeucOHHCcnR3penTp1vW3Xle\nffVVy1oCx3nR8rQO59DzIu66oiW6rK4WgfN0x48ZM2ZYLlWqlOW6devm6Hfo+aX3Ib3vOefcLbfc\nYlmHE+TXe0pO6BAMHb723XffWdbr29tvv+29X4fmVK5c2fKGDRss6/0o7tyJDpd74IEHLHfs2NFy\nMg2/SZ5PAgAAgKRFpxEAAABB+bI8rXTGps4A1HLX1KlTLX/++efe+7UMpwtr6uNrndGki0z37dvX\ncnTGp74/u2XoaEla5adSgv7vWLZsmeWDBw9ajhtG8NRTT3nH0rbRWc46+0xnmOmC6lo21VmDzvmz\n4HRIgv67His/ff/Zpd+xLt7cr18/yzocQMvLzvnfrZY19VzQ92hpRtt09+7d3nEzMjIst2vXzvL+\n/fstaxsX5DbKCj3voqsFLFq0yLJueqBZh+BoWU7bsWHDhon5sAWcDg84cOCAZR2CoX/TzvkLpceV\nJXW4js6yXb58uWVtR+ecGzBggOVUXxhfrxHt27e3rBs5fPLJJ5a17Zzzh7XpPUyz0nbUITq68YRz\nzg0ZMsRy9PqaLHjSCAAAgCA6jQAAAAjKl+VpfbSspUMtF7/zzjuWdXHg6ON+fVSsx9LyaIcOHTJ9\nfXZL0IkQt89lsqtevbrl4sWLW9aSsJZEo+UApY/ttZSjexzrgtw6a1CHGjjnD2/Q9tS/Ey3z5Kfv\nPCe0/KUzk3Vxc52J65w/W1BnCOr3qrOqdXaitmm0jXQfay2x6Uz2VGmX7NJhOs45N3nyZMs6TETP\nTx3yofsVv/LKK5Zzul9yQabfRVpamuXy5ctb1vJmdP/he+65x7KuFqCbFAwbNszytm3bLOt1S6+B\nzvllcNrrPL2+Dx061LJuyqF9C+ecW7p0qWW9jumx9FqnM+RffPFFy9E9rfU9iWyjRPYbeNIIAACA\nIDqNAAAACCp0oZm6uShXfqmWxLTEoqU23bfYOb/0lox7QV/k4t6J+vAJayedVbZ161bLupiqlju1\nbOycvxhtzZo1M32dtqXOpNbvLPr95XE7J+KX58q5pDOp9fzRdowuHK3fedzr9N/jFuaPnqNa8smD\nczRp2ygrdOFn55ybM2eO5Y8++siyLkKtM6O1pKoluugQAs6jmIPK9VsXftZZujoEwLn/bbP/p+eO\nft86/ENn444bN857fx4vgJ+0bZQV0VnR2kZ639J+hy7grte0uHtTEsjSh+FJIwAAAILoNAIAACCo\nQJWnYZKuPI1M5euSTYqgjZJfvmgjvdfqqhFvvPGG97opU6ZYPnbsmGUdfpWenm558ODBlmvXrm1Z\ny9ZJIF+0UYqjPA0AAIDEoNMIAACAIDqNAAAACGJMY8HEmMb8gXE+yY82Sn60UfKjjZIfYxoBAACQ\nGHQaAQAAEESnEQAAAEF0GgEAABBEpxEAAABBdBoBAAAQRKcRAAAAQXQaAQAAEJRXi3sDAAAgH+FJ\nIwAAAILoNAIAACCITiMAAACC6DQCAAAgiE4jAAAAgug0AgAAIIhOIwAAAILoNAIAACCITiMAAACC\n6DQCAAAgiE4jAAAAgug0AgAAIIhOIwAAAILoNAIAACCITiMAAACC6DQCAAAgiE4jAAAAgug0AgAA\nIIhOIwAAAILoNAIAACCITiMAAACC6DQCAAAgiE4jAAAAgug0AgAAIIhOIwAAAILoNAIAACDo/wCK\nORIASqOWeQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7c179dc50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAo0AAAB9CAYAAADHoFl9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFfZJREFUeJzt3XmwjmX8x/FLRUJROHZHdrJmLWTIVvRPyi5pGZlMo2Vi\nRo1JUxgjzBg1atAYIxM1qsFkm5EiYbJmiew7RWRL/f74/fr63NfvuV3ncJbnOc/79dfHOedZ3Ndz\n38819/daCv37778OAAAAuJHb8vsNAAAAIPnRaQQAAEAQnUYAAAAE0WkEAABAEJ1GAAAABNFpBAAA\nQBCdRgAAAATRaQQAAEAQnUYAAAAE3ZFPr8s2NLmrUA49D+2Uu3KinWij3EUbJT/aKPnRRskvS23E\nnUYAAAAE0WkEAABAUH6VpwEAAAq8f/+9XlkvVCinRo/lD+40AgAAIIhOIwAAAIIoTwMAgLR0M6Vj\nfYzmY8eOWS5cuLDlokWLWi5evLjl225Lvft2qfeOAQAAkOfoNAIAACCI8nQMveWclZ+n4m3mVKVt\n8M8//1i+evWq5ZMnT1q+/fbbLZcsWTLyXFoqQO64du2a5T179lh+//33LR84cMDygw8+GHn80KFD\nLZcuXdqy35b/SfXZiQDyzs1cL/SadvHiRcunTp2yvGzZMst//fWX5eHDh1suUaJEtl87v9HTAQAA\nQBCdRgAAAATRaQQAAEBQ2o1p9MckXrlyxfKOHTssT5o0yfLGjRst6/i4zMxMyw8//HDkeR977DHL\nderUsVykSJGbedtpT8crbtu2zfJrr71mWdvp3LlzlnW8aalSpSLPO3HiRMsDBgywrO2MrNFzS8f5\nrF+/3vIHH3xgedGiRZbvvPPOhI91zrlnnnnGctWqVS0zdjHnaNsdOnTIsp5rrVq1suyfR7QF0skd\nd1zvOun3y5dffml5xYoVlvv27WtZl99JRdxpBAAAQBCdRgAAAAQViltCJpfly4s6Fy1HOxctxYwc\nOdKy3lrWxxQrVixh9m85V6pUyfKIESMst2vXzrKW5HJYTtWKkqaddPmCF154wfLx48ct62c5bgkk\n/+cZGRmW161bZ7lixYrZfMc3JSfaKd/aSJc7cs65S5cuWdb2WrJkieUTJ05Y1mUodHiHDhNwzrny\n5ctb1nMmj0qiKd1GWaVt8cQTT1jWIR/Tpk2z3KtXr8jj83k4R4FqIz2vLly4YHn//v2Wx40bZ3nt\n2rWWtR30+8m56DIxOmRqyJAhllu3bm1ZzzV93ptcXq5AtZF+12zYsMFy7969Leuyb4MGDbKsQ9+0\nzJ0EstRG3GkEAABAEJ1GAAAABCXVvdHcoreS//7778jvdINx3ZVCb8eXKVPGct26dS3r7Wctc/u/\nGz16tOWPP/444XOxo0y0nf7888/I72bMmGFZy6DlypWz3LlzZ8s6e/2rr76yvHDhwsjzanlbX2PU\nqFGWmRl6XdwMaeecW758uWU9ztWqVbM8bNiwhD8vXLiwZf9c0ONPW+QcLVfqZ1+H5uiOFXqu+NdR\nVhu4ef4QMb32TZkyxfL06dMt6/eWtqPyzyN9ne3bt1tevXq15RdffNHy448/bll3adJz1bn0PCf1\nWOp3vQ6/0SFWer7oz5OsPJ0l9FQAAAAQRKcRAAAAQal3b/QW+TM+V61aZfno0aOW7777bss647lG\njRqWdTablm6ci5ZQ9+zZY1lLpTVr1rTMot9RfjtVqVLFcteuXS3379/fcseOHS3r8Wzfvr3l77//\nPvK8Oqxgy5Ytt/CO04OWZXbv3h35nZY4q1evblkX565QoYJlLc3cqMSVjuWvvKBlsqVLl1quX7++\n5Z49e1pu27atZdok5/jl6dOnT1teuXKl5fPnz1vW4QBx+a677oo8r67wcfnyZct6rmq7+rOvE/1N\nutLvJ509rUMFdHjA77//bvnw4cOW9dg7lxrDPLjTCAAAgCA6jQAAAAhKi/K03v7XErRzzn300UeW\nz5w5Y/nee++1/McffyR83oYNG1rWcrRz0UVx9fXjZlTpjLR0vf2v/2+/tFK7dm3LWqpu06aN5biF\nn/WY657UzkXbpmzZsjfzttOKfmb9Ur/O6NRSpp5LWrJhVnTe08/7nDlzLOse0926dbPcokULy1re\n9IeP4Ob5GxlMnTrVss5y1mN+zz33WNYSZ5cuXSzrfsfOOVe5cmXLZ8+eTfi8Wl7VIVq0d9SpU6cs\n66x2Lfvr+aL9Dt3sYPDgwZHnLV68uOVkvSZypxEAAABBdBoBAAAQlBbl6atXr1oeO3Zs5HdHjhyx\nrKUbne2kZWstzzVr1sxykyZNIs+rs950EeRatWolfF9aWo0r4aUTf1/uhx56yHLp0qUtx83w05LP\nvHnzLOseu85Fj6++Bq6LG17hL2ifmZlpWRcD1hmB+bTXPf6PHn8tseksaS1P60xezbryA7JP28Ff\neUNX2NCFvvWa2KpVK8u6d3SnTp0s+yty6PdKXOlZh+/o32vZ2l/YPRUXqM4u/a52zrnx48db1mE5\neix0pQj93tfFwPft2xd53nr16llO1pnU3GkEAABAEJ1GAAAABBXY+8px+xjrrX/n/v9t50R0lnSv\nXr0saznOL7vpbF9dBPzChQsJX4OZpFF+yUOPZ9wMdG2DgwcPWl6wYEHs69x3332WdRFw2uA6PRY6\ny19najoXXTFAZ+PqeaL7uGuJjOOd91q3bm1Zy4+637TO9KxTp45lhnLcGi0JT5s2LfI7/b7Qa5rO\nrG3ZsqVlbRctad7MOaWrVuhr6/X4RntaF6TzWP9fuiC3c87NnTs34WN0ZY8xY8ZY1nbZtWuX5XXr\n1kUeX7VqVcs6Qz6ZcKcRAAAAQXQaAQAAEESnEQAAAEEFdkyjjhkZNWqU5bjdXZyLjtvo3r275REj\nRljOyMiwrOO7/DGNOrZDd8RQLD8Szx8bo8dax4XqEjonTpywrOOEdFcFX8eOHS2zI0yYLvvRrl27\nyO907OLnn39uefbs2ZZ1SRAdU+ePk0Lu0zGleu3bvHmzZV16rFKlSpaTdTmQZKbXe13G7ZNPPon8\nnY7T1uugHvMNGzYk/PsePXpY9pdF8nfZSvS8WRlb7/+8II1jVNqH0OuZc9EddfTc6dOnj2Ud96vz\nKnSZvwMHDkSeV/snyTrmmys1AAAAgug0AgAAIKjAlqcvXbpkefHixZb9krDeWn722WctT5o0yXJW\nNhH3N3TXJSx+/fXXhM+lJTlK1Tem7aR5y5YtlidMmGB51apVlnUTeX8HmX79+lnWEjjC/GWRqlWr\nZvmpp56yPH/+fMs//vij5aZNm1q+1aVCkDV6XdLylx7/PXv2WNbdrHQnJtoo+/Qav3TpUstaurzR\nY7SkvWzZMst6rdPvugceeCDyXNWrV7fcuHFjy7p7k34/6ZIv6bhLmX5vzJgxI/I7XaqvVKlSlhs0\naJDw51rOXrFihWV/R5iKFStaHjRokGV/h7T8xJ1GAAAABNFpBAAAQFCBKk/rrfzdu3db1hlJfknt\n6aeftjxlyhTLfhkzxL9lr7MOFy1aZLlcuXKWdacZZo9mnQ49+Prrry2vWbPGss6q1tJbo0aNIs/V\noUMHy+lSdskt+hnWUvWjjz5q+bPPPrN86NAhy7Vq1bJMO+QeLbnpOaKzOHUVAt0xSUtntFH26fdT\n+fLlLfs7f+jqEDrjWUuUOpRG20JnUvszc48fP25Zz71NmzZZ7tq1q2UtW6dLe2sbrV+/3rJfRtbj\noaX+tm3bWtbroR7juNUJnHNu8uTJlps3b25Zh/Lkd1vQUwEAAEAQnUYAAAAEFdjy9KeffprwbypX\nrhz59zvvvGM5bvHTrPBnTy9YsMDyL7/8YvngwYOWtVwQtwA4/peWXfR4fvvtt5bPnz+f8LE6S7RZ\ns2aR3zFrN3foca1Ro4ZlLY/qTE8tZyfTTMFU56/KoKXnHTt2WN66davlkydPWh44cKDlkiVL5sZb\nTBt6fdGZzcOGDYv8nQ5h0hKlOnfunGUtb+twgp9//jnymIULF1rW7yt9Lr2G6t+ky7XxypUrlidO\nnJjw585F+wq6oHeZMmUs63eWfr9rG506dSryvMeOHbM8c+ZMyzqULb9X+eBOIwAAAILoNAIAACCo\nQJWndVatljB1xnSvXr0ij9FydXZvwWvpxy+Nbtu2zfL+/fsta6nUL2njOr+spgukv/7665a1nbX9\nSpQoYbl+/fqW/WEAWi7VkgOz2XOOHkstPe/du9eyzuQtUqRI5PHpUhrLC3GlMd0fvF69epbjZuki\na+I2bdB2ePPNNyO/089/3HVI20JfQ79TdKFu56LXPh3Wo4uL6/mZLvuL6/HTkr4umu4fi06dOlnu\n2bNnwufV8rT2D1q3bm356NGjkcdoe58+fdqy9m0oTwMAACDp0WkEAABAUMqXp/XW8vLlyy3/9NNP\nlitUqGBZ98R1LnrbWZ8rrhSjf6O3jHWmk3POfffdd5Z1lrTOdNPbzHFljHQqCekxOHLkSOR3Oqxg\n586dlvV2fvv27S1ryeBGJRddALdo0aKWtVSdTm2QU7QtdXH9vn37Jvy5niN+Wc0vV+Pm6VAdXeBZ\nS2l6Huk5oSU2/blznCNKS8RxM5D1uuWv2pHdY6l/r9mf7a7fg7qotK7oEfceCzLdR1pXU9EZ0/6x\nfPXVVy3758J/9LumZs2alnft2mXZv7bpeRg37CC/pcenAgAAALeETiMAAACCUr48rSXikSNHWtZS\nipYn/VvJ165dS/i8+nN9Dd3TetasWZZ1MW/nnDt79qxlvU1du3bthO9Lb0trCSmdyj662LDuV+xc\ntHypZZMuXbpYnjt3rmUt+cTtsZvo3/9p0KCBZX+/coRpyUdLMDozV2eua7lMy9bORWea0hY5RxdU\nf/vtty3rORW3GHibNm1y982lGC0f6nePfq7Lli1rWb+Hcusa7z+vllt1ZrAOB8nMzMz195VsdHWT\ntWvXWtbrk67G4Vz0+1q/63XIme4hvnHjRsvffPON5TNnzkSeV/sdOkQrma573GkEAABAEJ1GAAAA\nBNFpBAAAQFDyFMqzQaei64rqhw8ftqxjAHQFdh3H49OxVGvWrLE8evTohK+nK+nrGC6fvqaOldOx\nWjpOL13GkjgXHf/z8ssvW9axVM5Fx3/26NHD8vTp0y3ruBM9njpmR5c8cs65rVu3Wl6/fr3ljIwM\ny7pURTq1TXbFLRGhuyDpmB89R/WxFy9ejDyvjg3S9qMtsk/HtZUpU8by/fffb1nHA+vfb9++3TJj\nGqN0jNvKlSst6/Wlf//+lqtUqZLr70nbzjnn3n33XcvHjh2z3LRpU8u6tEy6nF/6na7tqP9//blz\nzo0fP95yy5YtLZcrV86yjo/U5QB1Jyaffs9169bNcjLtzsOdRgAAAATRaQQAAEBQypendakbLX3p\n5uwjRoywfKMV2NetW2dZV3zXKfn62vpc/i4WlSpVsqw7LAwaNCjh+02XUoBz0WOuK/Dr8fc98sgj\nlidMmGC5WLFilrVtdOkCLdP4S+ysWLHCspaStG3085NMSx/kN3+Xgrjlq/Q80c+5lly0hK0lGuei\nw0b0ufJi2ZJU57fR6dOnLVetWtWy7lih9Dzwdy5BYrq8yg8//GBZr09vvPGG5ZwsPWp7L168OPK7\nZcuWWdZzdejQoZb1epoutLysJeElS5ZY9ofM6FAm3eElbrmluB1d/KV8BgwYYHnw4MGWk2lXLO40\nAgAAIIhOIwAAAIJSstam5UYtaeot3Dp16iR8rM54ds65Dz/80PLUqVMtnzx50rKWFbR01qpVK8ta\ngnYuOmO7UaNGlnV2WtzOL/p6mp0rGJvI79y50/KcOXMs684t/gbxlStXtrx3717LWgLQttUyge5k\noavx++9F20BX+dfPm5aSCnJJNO4zqKsE6E5JzsXvgqTHSc8fLc3oMdbhIM5FZyE2bNjQcosWLRK+\nBuLpcda20DbWv9m3b59lbS+Od5Rel/VYbtu2zbLOWH7uuecs6+oMzmX/2GrpU3d60bKzc9ESq85+\nT9ZZunlFh7lMmzbN8ksvvWR59erVkcecO3fOsn7v+LOs/6Pf9XXr1rWsQ62cc65du3aWdahAMp1v\nqd8DAQAAQK6j0wgAAICglCxP6+3448ePW9ZFgLWEOWTIkIQ/dy66sKc/Q+o/WirVBVpfeeUVyxUr\nVow8Jq48pz+PWwxZFYRytHPR2Xrz58+3rLM59Xhoedm56Ew2XXhdywHa/no8ddiCv+Ctvi9twyZN\nmlhOl1nuesz0uB46dMjy5s2bLfvn0pkzZyz/9ttvlrVsrSVRXfRe/0ZnJjrn3KZNmywPHz7csg77\nYGZvYv515eDBg5b1s6znmy4+rG0/cOBAywXlupRT9Lpevnx5yzrzX691EydOtPzee+9FnstfPSAR\nvY5NmTLF8tixYy3711Atg2sZ1p/Bm850wfvZs2dbXrRoUeTv5s2bZ1mHz2jZWjfvmDRpkuXOnTtb\n9ts6FYYHcOYDAAAgiE4jAAAAggr5s3PzyC29qJbOJk+ebHnMmDGW/ZmdcXRWk5a4+vTpY3nUqFGW\n9fZ13Ozn3JTF18mpN5NjHw4tk33xxReWhw0bZlnLYn75S2/bx5X+4xZ+rl27tmVtP+ece/LJJy3r\nLEJdbDoXF/TOiXbKsTbSa8Hly5cta4ly9+7dlnXhYueiexNr6VrbqHTp0pZ1trwOLdDX9h+je742\nb97cclZKejcpqdoou3S2u3POzZo1y7IuSqzlTj1X9fzQmZ1Jtsh9UrWRrryh1x4dgqHXtw4dOkQe\nH7fwt55TM2fOtKyLiet3o79Q99y5cy13797dch59dyVVG2X7hb1+kp4jcZsaaBunyKobWXpj3GkE\nAABAEJ1GAAAABKVkeVpLLjobcOTIkZZ1H2Hdh1pnLjnn3PPPP285IyPDctx+ubklbjFl/7VTtTyt\ntP0OHz5sWfeF1nKlc9H2iJvNrLMWtbys2d/DMz+GGIikLdlo+UWzfjb9hWz1+MU9XulqBdqmfhvp\nv/OhzJO0bZQV/rHXmejjxo2zrOdev379LOswHR3akWQltqRqIz1HdP/n3r17W9brm/8drMc2btOH\nuOOvM3aXLl0a+V3jxo2Dj89FSdVGSIjyNAAAAHIGnUYAAAAEpWR5OvZJs3D7Pk0kdXkahpJN8itQ\nbaRDQ3Qh4hMnTljOzMy0rKsQJPGC3knbRvqdpBtR6IocWsJ2LjrLWttLn0sX6n7rrbcs6wLsSbbg\nfdK2EQzlaQAAAOQMOo0AAAAIotMIAACAoAI1phGGMY2pgXE+yY82Sn60UfKjjZIfYxoBAACQM+g0\nAgAAIIhOIwAAAILoNAIAACCITiMAAACC6DQCAAAgiE4jAAAAgug0AgAAICi/FvcGAABACuFOIwAA\nAILoNAIAACCITiMAAACC6DQCAAAgiE4jAAAAgug0AgAAIIhOIwAAAILoNAIAACCITiMAAACC6DQC\nAAAgiE4jAAAAgug0AgAAIIhOIwAAAILoNAIAACCITiMAAACC6DQCAAAgiE4jAAAAgug0AgAAIIhO\nIwAAAILoNAIAACCITiMAAACC6DQCAAAgiE4jAAAAgug0AgAAIIhOIwAAAILoNAIAACDofwDMzy/w\npqgG7QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b7bf537fd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_iterations = 3\n",
    "n_digits = 6\n",
    "codings_rnd = np.random.normal(size=[n_digits, n_hidden3])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"./my_model_variational.ckpt\")\n",
    "    target_codings = np.roll(codings_rnd, -1, axis=0)\n",
    "    for iteration in range(n_iterations + 1):\n",
    "        codings_interpolate = codings_rnd + (target_codings - codings_rnd) * iteration / n_iterations\n",
    "        outputs_val = outputs.eval(feed_dict={codings: codings_interpolate})\n",
    "        plt.figure(figsize=(11, 1.5*n_iterations))\n",
    "        for digit_index in range(n_digits):\n",
    "            plt.subplot(1, n_digits, digit_index + 1)\n",
    "            plot_image(outputs_val[digit_index])\n",
    "        plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
